back at it again

peertube/statnerd/main.py

@@ -0,0 +1,119 @@
+import schedule
+import signal
+import json
+import time
+import socket
+import logging
+from utils.ColoredFormatter import ColoredFormatter
+from bs4 import BeautifulSoup as bs
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.common.by import By
+from selenium.webdriver import ActionChains
+from selenium.webdriver.support.wait import WebDriverWait
+from selenium.webdriver.support import expected_conditions as ec
+
+def setupLogger():
+    logging_format = "[%(asctime)s] (%(levelname)s) %(module)s - %(funcName)s: %(message)s"
+    logging.basicConfig(level=logging.INFO, format=logging_format)
+    (logger := logging.getLogger(__name__)).setLevel(logging.INFO)
+    logger.propagate = False
+    (logger_handler := logging.StreamHandler()).setFormatter(
+        ColoredFormatter(fmt=logging_format)
+    )
+    logger.addHandler(logger_handler)
+
+def interrupt_handler(signum, driver: webdriver.Chrome):
+    print(f'Handling signal {signum} ({signal.Signals(signum).name}).')
+
+    schedule.clear()
+    driver.quit()
+    raise SystemExit
+
+def setupChromeDriver():
+    logging.log(logging.CRITICAL, 'Setting up Chrome driver.')
+    chrome_options = Options()
+    chrome_options.add_argument("--headless")
+    chrome_options.add_argument("--no-sandbox")
+    chrome_options.add_argument("--mute-audio")
+    chrome_options.add_argument("--window-size=1280,720")
+    chrome_options.add_argument("--disable-dev-shm-usage")
+    chrome_options.add_argument("--disable-features=WebRtcHideLocalIpsWithMdns")
+    chrome_options.add_experimental_option('prefs', {'intl.accept_languages': 'en,en_US'})
+    #chrome_options.add_extension('./qryn-webrtc-exporter.crx')
+
+    driver = webdriver.Chrome(options=chrome_options)
+    #driver = webdriver.Remote(command_executor='http://localhost:4444', options=chrome_options)
+    return driver
+
+def saveStats(stats: dict):
+    try:
+        sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+        sock.sendto(json.dumps(stats).encode(), ('localhost', 8094))
+        sock.close()
+        print(f'Sent stats: {stats}')
+    except socket.error as e:
+        print(f'Got socket error: {e}')
+
+def downloadStats(driver: webdriver.Chrome):
+    html = driver.find_element(By.CLASS_NAME ,'vjs-stats-list').get_attribute('innerHTML')
+    if html is not None:
+        htmlBS = bs(html, 'html.parser')
+    else:
+        raise ValueError("html is None")
+    
+    stats = htmlBS.find_all('div', attrs={'style': 'display: block;'})
+
+    statsDict = {
+        stat.div.text: stat.span.text.replace('\u21d3', 'down').replace('down/', 'down /').replace('\u21d1 ', 'up').replace('\u21d1', 'up').replace('\u00b7', '-').strip()
+        for stat in stats
+    }
+    
+    for stat in statsDict:
+        if 'Buffer State' == stat:
+            statsDict[stat] = statsDict[stat][1:-1].split(', ')
+
+    # statsDict = {
+    # 'userName': dict(
+    #     map(
+    #         lambda stat: (
+    #                 stat.div.text, 
+    #                 stat.span.text.replace('\u21d3', 'down').replace('down/', 'down /').replace('\u21d1 ', 'up').replace('\u21d1', 'up').replace('\u00b7', '-').strip()
+    #             ), stats
+    #         )
+    #     )
+    # }
+
+    statsDict.update({'Timestamp': time.strftime('%Y-%m-%dT%H:%M:%S%z')})
+    statsDict['userName'] = 'user'
+
+    saveStats(statsDict)
+    
+def setupStats(driver: webdriver.Chrome, url: str):
+    actions = ActionChains(driver)
+    wait = WebDriverWait(driver, 30, poll_frequency=0.2)
+
+    driver.get(url)
+
+    wait.until(ec.presence_of_element_located((By.CLASS_NAME, 'vjs-big-play-button')))
+    actions.click(driver.find_element(By.CLASS_NAME ,'video-js')).perform()
+    actions.context_click(driver.find_element(By.CLASS_NAME ,'video-js')).perform()
+    statsForNerds = driver.find_elements(By.CLASS_NAME ,'vjs-menu-item')
+    actions.pause(2)
+    actions.click(statsForNerds[-1]).perform()
+    wait.until(ec.text_to_be_present_in_element((By.CLASS_NAME, 'vjs-stats-list'), 'Player'))
+
+    return driver
+
+if __name__ == '__main__':    
+    setupLogger()
+    
+    driver = setupChromeDriver()
+
+    signal.signal(signal.SIGINT, lambda signum, frame: interrupt_handler(signum, driver))
+    
+    setupStats(driver, "https://tube.kobim.cloud/w/9hAbiwai4rsbw9QnPpPkCd")
+    
+    schedule.every(1).seconds.do(downloadStats, driver)
+    while True:
+        schedule.run_pending()

peertube/statnerd/requirements.txt

@@ -1,3 +1,3 @@
-jupyterlab==4.1.2
+jupyterlab
-selenium==4.18.1
+selenium
 schedule

peertube/statnerd/statistiche.ipynb

@@ -19,6 +19,7 @@
     "import signal\n",
     "import json\n",
     "import time\n",
+    "import socket\n",
     "from bs4 import BeautifulSoup as bs\n",
     "from selenium import webdriver\n",
     "from selenium.webdriver.chrome.options import Options\n",
@@ -26,7 +27,7 @@
     "from selenium.webdriver import ActionChains\n",
     "from selenium.webdriver.support.wait import WebDriverWait\n",
     "from selenium.webdriver.support import expected_conditions as ec\n",
-    "from IPython.display import display, display_html, DisplayHandle, Image"
+    "from IPython.display import display, DisplayHandle, Image"
    ]
   },
   {
@@ -55,9 +56,16 @@
     "def setupChromeDriver():\n",
     "    chrome_options = Options()\n",
     "    chrome_options.add_argument(\"--headless\")\n",
-    "    chrome_options.add_argument(\"--window-size=1280,720\")\n",
+    "    chrome_options.add_argument(\"--no-sandbox\")\n",
     "    chrome_options.add_argument(\"--mute-audio\")\n",
+    "    chrome_options.add_argument(\"--window-size=1280,720\")\n",
+    "    chrome_options.add_argument(\"--disable-dev-shm-usage\")\n",
+    "    chrome_options.add_argument(\"--disable-features=WebRtcHideLocalIpsWithMdns\")\n",
+    "    chrome_options.add_experimental_option('prefs', {'intl.accept_languages': 'en,en_US'})\n",
+    "    #chrome_options.add_extension('./qryn-webrtc-exporter.crx')\n",
+    "\n",
     "    driver = webdriver.Chrome(options=chrome_options)\n",
+    "    #driver = webdriver.Remote(command_executor='http://localhost:4444', options=chrome_options)\n",
     "    return driver"
    ]
   },
@@ -68,16 +76,13 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "def saveStats(stats):\n",
+    "def saveStats(stats: dict):\n",
-    "\n",
+    "    try:\n",
-    "    with open('stats.json', 'r+') as jsonFile:\n",
+    "        sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)\n",
-    "        try:\n",
+    "        sock.sendto(json.dumps(stats).encode(), ('localhost', 8094))\n",
-    "            data = json.load(jsonFile)\n",
+    "        sock.close()\n",
-    "        except json.JSONDecodeError:\n",
+    "    except socket.error as e:\n",
-    "            data = []\n",
+    "        print(f'Got socket error: {e}')\n",
-    "        data.append(stats)\n",
-    "        jsonFile.seek(0)\n",
-    "        json.dump(data, jsonFile, indent=4)\n",
     "\n",
     "def downloadStats(driver: webdriver.Chrome, display_handle: DisplayHandle):\n",
     "    html = driver.find_element(By.CLASS_NAME ,'vjs-stats-list')\n",
@@ -103,18 +108,33 @@
     "    htmlBS.div.insert_before(peersDiv)\n",
     "    \n",
     "    stats = htmlBS.find_all('div', attrs={'style': 'display: block;'})\n",
-    "    statsList = dict(\n",
-    "        map(\n",
-    "            lambda stat: (\n",
-    "                stat.div.text, \n",
-    "                stat.span.text.replace('\\u21d3', 'down').replace('down/', 'down /').replace('\\u21d1 ', 'up').replace('\\u21d1', 'up').replace('\\u00b7', '-').strip()\n",
-    "            ), stats\n",
-    "        )\n",
-    "    )\n",
     "\n",
-    "    display_handle.update(statsList)\n",
+    "    statsDict = {\n",
+    "        stat.div.text: stat.span.text.replace('\\u21d3', 'down').replace('down/', 'down /').replace('\\u21d1 ', 'up').replace('\\u21d1', 'up').replace('\\u00b7', '-').strip()\n",
+    "        for stat in stats\n",
+    "    }\n",
+    "    \n",
+    "    for stat in statsDict:\n",
+    "        if 'Buffer State' == stat:\n",
+    "            statsDict[stat] = statsDict[stat][1:-1].split(', ')\n",
     "\n",
-    "    saveStats(statsList)"
+    "    # statsDict = {\n",
+    "    # 'userName': dict(\n",
+    "    #     map(\n",
+    "    #         lambda stat: (\n",
+    "    #                 stat.div.text, \n",
+    "    #                 stat.span.text.replace('\\u21d3', 'down').replace('down/', 'down /').replace('\\u21d1 ', 'up').replace('\\u21d1', 'up').replace('\\u00b7', '-').strip()\n",
+    "    #             ), stats\n",
+    "    #         )\n",
+    "    #     )\n",
+    "    # }\n",
+    "\n",
+    "    statsDict.update({'Timestamp': time.strftime('%Y-%m-%dT%H:%M:%S%z')})\n",
+    "    statsDict['userName'] = 'user'\n",
+    "\n",
+    "    display_handle.update(json.dumps(statsDict))\n",
+    "\n",
+    "    saveStats(statsDict)"
    ]
   },
   {
@@ -134,7 +154,7 @@
     "    actions.click(driver.find_element(By.CLASS_NAME ,'video-js')).perform()\n",
     "    actions.context_click(driver.find_element(By.CLASS_NAME ,'video-js')).perform()\n",
     "    statsForNerds = driver.find_elements(By.CLASS_NAME ,'vjs-menu-item')\n",
-    "    actions.pause(1)\n",
+    "    actions.pause(2)\n",
     "    actions.click(statsForNerds[-1]).perform()\n",
     "    wait.until(ec.text_to_be_present_in_element((By.CLASS_NAME, 'vjs-stats-list'), 'Player'))\n",
     "    actions.move_to_element(driver.find_element(By.CLASS_NAME ,'vjs-peertube')).perform()\n",
@@ -158,11 +178,11 @@
     "\n",
     "    signal.signal(signal.SIGINT, lambda signum, frame: interrupt_handler(signum, driver))\n",
     "    \n",
-    "    setupStats(driver, \"https://tube.kobim.cloud/w/gFL48Fz3doCEnYwK46BwYN\")\n",
+    "    setupStats(driver, \"https://tube.kobim.cloud/w/9hAbiwai4rsbw9QnPpPkCd\")\n",
     "\n",
     "    display_handle = display(\"Loading...\", display_id=True)\n",
     "    \n",
-    "    schedule.every(0.3).seconds.do(downloadStats, driver, display_handle)\n",
+    "    schedule.every(1).seconds.do(downloadStats, driver, display_handle)\n",
     "    while True:\n",
     "        schedule.run_pending()"
    ]
@@ -184,7 +204,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.3"
+   "version": "3.12.5"
   }
  },
  "nbformat": 4,

peertube/statnerd/telegraf.conf

@@ -0,0 +1,60 @@
+[[processors.dedup]]
+    dedup_interval = "600s"
+
+[[outputs.file]]
+    files = ["stdout"]
+
+# [[outputs.mongodb]]
+#     dsn = "mongodb://192.168.86.40:27017"
+#     database = "peertube"
+#     granularity = "seconds"
+#     authentication = "SCRAM"
+#     username = "root"
+#     password = "example"
+
+[[inputs.socket_listener]]
+    service_address = "udp://:8094"
+    data_format = "xpath_json"
+    [[inputs.socket_listener.xpath]]
+        metric_name = "'stats'"
+
+        [inputs.socket_listener.xpath.fields]
+            P2P = "/P2P"
+            #Buffer_State = "string-join(/Buffer_State/*, ',')"
+            Buffer_State = "/text()='Buffer State'"
+            Resolution = "/Resolution"
+
+# [[inputs.socket_listener]]
+#     service_address = "udp://:8094"
+#     data_format = "json_v2"
+#     [[inputs.socket_listener.json_v2]]
+#         measurement_name = "stats"
+
+#         [[inputs.socket_listener.json_v2.field]]
+#             path = "P2P"
+#         [[inputs.socket_listener.json_v2.field]]
+#             path = "Player mode"
+#         [[inputs.socket_listener.json_v2.field]]
+#             path = "P2P"
+#         [[inputs.socket_listener.json_v2.field]]
+#             path = "Video UUID"
+#         [[inputs.socket_listener.json_v2.field]]
+#             path = "Viewport / Frames"
+#         [[inputs.socket_listener.json_v2.field]]
+#             path = "Resolution"
+#         [[inputs.socket_listener.json_v2.field]]
+#             path = "Volume"
+#         [[inputs.socket_listener.json_v2.field]]
+#             path = "Codecs"
+#         [[inputs.socket_listener.json_v2.field]]
+#             path = "Connection Speed"
+#         [[inputs.socket_listener.json_v2.field]]
+#             path = "Network Activity"
+#         [[inputs.socket_listener.json_v2.field]]
+#             path = "Total Transfered"
+#         [[inputs.socket_listener.json_v2.field]]
+#             path = "Download Breakdown"
+#         [[inputs.socket_listener.json_v2.field]]
+#             path = "Buffer State"
+#         [[inputs.socket_listener.json_v2.field]]
+#             path = "Live Latency"

peertube/statnerd/utils/ColoredFormatter.py

@@ -0,0 +1,43 @@
+import logging
+
+class ColoredFormatter(logging.Formatter):
+    """Colored formatter for the logging package."""
+
+    def __init__(
+        self, fmt=None, datefmt=None, style="%", validate=True, *, defaults=None
+    ):
+        """Colored formatter for the logging package."""
+        fmt = fmt or "%(levelname)s: %(message)s"
+        super().__init__(fmt, datefmt, style, validate, defaults=defaults)
+        colors = {
+            "red": "\x1b[31;20m",
+            "bold_red": "\x1b[31;1m",
+            "green": "\x1b[32;20m",
+            "bold_green": "\x1b[32;1m",
+            "yellow": "\x1b[33;20m",
+            "bold_yellow": "\x1b[33;1m",
+            "blue": "\x1b[34;20m",
+            "bold_blue": "\x1b[34;1m",
+            "grey": "\x1b[37;20m",
+            "bold_grey": "\x1b[37;1m",
+            "reset": "\x1b[0m",
+        }
+        self._default_formatter = logging.Formatter(fmt)
+        self._formatters = {
+            100: logging.Formatter(colors["bold_blue"] + fmt + colors["reset"]),
+            logging.DEBUG: logging.Formatter(colors["grey"] + fmt + colors["reset"]),
+            logging.INFO: logging.Formatter(colors["green"] + fmt + colors["reset"]),
+            logging.WARNING: logging.Formatter(
+                colors["yellow"] + fmt + colors["reset"]
+            ),
+            logging.ERROR: logging.Formatter(colors["red"] + fmt + colors["reset"]),
+            logging.CRITICAL: logging.Formatter(
+                colors["bold_red"] + fmt + colors["reset"]
+            ),
+        }
+
+    def format(self, record):
+        """Override of logging.Formatter.format"""
+        return self._formatters.get(record.levelno, self._default_formatter).format(
+            record
+        )