diff --git a/peertube/statnerd/main.py b/peertube/statnerd/main.py new file mode 100644 index 0000000..913f076 --- /dev/null +++ b/peertube/statnerd/main.py @@ -0,0 +1,119 @@ +import schedule +import signal +import json +import time +import socket +import logging +from utils.ColoredFormatter import ColoredFormatter +from bs4 import BeautifulSoup as bs +from selenium import webdriver +from selenium.webdriver.chrome.options import Options +from selenium.webdriver.common.by import By +from selenium.webdriver import ActionChains +from selenium.webdriver.support.wait import WebDriverWait +from selenium.webdriver.support import expected_conditions as ec + +def setupLogger(): + logging_format = "[%(asctime)s] (%(levelname)s) %(module)s - %(funcName)s: %(message)s" + logging.basicConfig(level=logging.INFO, format=logging_format) + (logger := logging.getLogger(__name__)).setLevel(logging.INFO) + logger.propagate = False + (logger_handler := logging.StreamHandler()).setFormatter( + ColoredFormatter(fmt=logging_format) + ) + logger.addHandler(logger_handler) + +def interrupt_handler(signum, driver: webdriver.Chrome): + print(f'Handling signal {signum} ({signal.Signals(signum).name}).') + + schedule.clear() + driver.quit() + raise SystemExit + +def setupChromeDriver(): + logging.log(logging.CRITICAL, 'Setting up Chrome driver.') + chrome_options = Options() + chrome_options.add_argument("--headless") + chrome_options.add_argument("--no-sandbox") + chrome_options.add_argument("--mute-audio") + chrome_options.add_argument("--window-size=1280,720") + chrome_options.add_argument("--disable-dev-shm-usage") + chrome_options.add_argument("--disable-features=WebRtcHideLocalIpsWithMdns") + chrome_options.add_experimental_option('prefs', {'intl.accept_languages': 'en,en_US'}) + #chrome_options.add_extension('./qryn-webrtc-exporter.crx') + + driver = webdriver.Chrome(options=chrome_options) + #driver = webdriver.Remote(command_executor='http://localhost:4444', options=chrome_options) + return driver + +def saveStats(stats: dict): + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.sendto(json.dumps(stats).encode(), ('localhost', 8094)) + sock.close() + print(f'Sent stats: {stats}') + except socket.error as e: + print(f'Got socket error: {e}') + +def downloadStats(driver: webdriver.Chrome): + html = driver.find_element(By.CLASS_NAME ,'vjs-stats-list').get_attribute('innerHTML') + if html is not None: + htmlBS = bs(html, 'html.parser') + else: + raise ValueError("html is None") + + stats = htmlBS.find_all('div', attrs={'style': 'display: block;'}) + + statsDict = { + stat.div.text: stat.span.text.replace('\u21d3', 'down').replace('down/', 'down /').replace('\u21d1 ', 'up').replace('\u21d1', 'up').replace('\u00b7', '-').strip() + for stat in stats + } + + for stat in statsDict: + if 'Buffer State' == stat: + statsDict[stat] = statsDict[stat][1:-1].split(', ') + + # statsDict = { + # 'userName': dict( + # map( + # lambda stat: ( + # stat.div.text, + # stat.span.text.replace('\u21d3', 'down').replace('down/', 'down /').replace('\u21d1 ', 'up').replace('\u21d1', 'up').replace('\u00b7', '-').strip() + # ), stats + # ) + # ) + # } + + statsDict.update({'Timestamp': time.strftime('%Y-%m-%dT%H:%M:%S%z')}) + statsDict['userName'] = 'user' + + saveStats(statsDict) + +def setupStats(driver: webdriver.Chrome, url: str): + actions = ActionChains(driver) + wait = WebDriverWait(driver, 30, poll_frequency=0.2) + + driver.get(url) + + wait.until(ec.presence_of_element_located((By.CLASS_NAME, 'vjs-big-play-button'))) + actions.click(driver.find_element(By.CLASS_NAME ,'video-js')).perform() + actions.context_click(driver.find_element(By.CLASS_NAME ,'video-js')).perform() + statsForNerds = driver.find_elements(By.CLASS_NAME ,'vjs-menu-item') + actions.pause(2) + actions.click(statsForNerds[-1]).perform() + wait.until(ec.text_to_be_present_in_element((By.CLASS_NAME, 'vjs-stats-list'), 'Player')) + + return driver + +if __name__ == '__main__': + setupLogger() + + driver = setupChromeDriver() + + signal.signal(signal.SIGINT, lambda signum, frame: interrupt_handler(signum, driver)) + + setupStats(driver, "https://tube.kobim.cloud/w/9hAbiwai4rsbw9QnPpPkCd") + + schedule.every(1).seconds.do(downloadStats, driver) + while True: + schedule.run_pending() \ No newline at end of file diff --git a/peertube/statnerd/qryn-webrtc-exporter.crx b/peertube/statnerd/qryn-webrtc-exporter.crx new file mode 100644 index 0000000..03ae125 Binary files /dev/null and b/peertube/statnerd/qryn-webrtc-exporter.crx differ diff --git a/peertube/statnerd/requirements.txt b/peertube/statnerd/requirements.txt index d136a82..0f46d71 100644 --- a/peertube/statnerd/requirements.txt +++ b/peertube/statnerd/requirements.txt @@ -1,3 +1,3 @@ -jupyterlab==4.1.2 -selenium==4.18.1 +jupyterlab +selenium schedule \ No newline at end of file diff --git a/peertube/statnerd/statistiche.ipynb b/peertube/statnerd/statistiche.ipynb index 6fb5b52..673c512 100644 --- a/peertube/statnerd/statistiche.ipynb +++ b/peertube/statnerd/statistiche.ipynb @@ -19,6 +19,7 @@ "import signal\n", "import json\n", "import time\n", + "import socket\n", "from bs4 import BeautifulSoup as bs\n", "from selenium import webdriver\n", "from selenium.webdriver.chrome.options import Options\n", @@ -26,7 +27,7 @@ "from selenium.webdriver import ActionChains\n", "from selenium.webdriver.support.wait import WebDriverWait\n", "from selenium.webdriver.support import expected_conditions as ec\n", - "from IPython.display import display, display_html, DisplayHandle, Image" + "from IPython.display import display, DisplayHandle, Image" ] }, { @@ -55,9 +56,16 @@ "def setupChromeDriver():\n", " chrome_options = Options()\n", " chrome_options.add_argument(\"--headless\")\n", - " chrome_options.add_argument(\"--window-size=1280,720\")\n", + " chrome_options.add_argument(\"--no-sandbox\")\n", " chrome_options.add_argument(\"--mute-audio\")\n", + " chrome_options.add_argument(\"--window-size=1280,720\")\n", + " chrome_options.add_argument(\"--disable-dev-shm-usage\")\n", + " chrome_options.add_argument(\"--disable-features=WebRtcHideLocalIpsWithMdns\")\n", + " chrome_options.add_experimental_option('prefs', {'intl.accept_languages': 'en,en_US'})\n", + " #chrome_options.add_extension('./qryn-webrtc-exporter.crx')\n", + "\n", " driver = webdriver.Chrome(options=chrome_options)\n", + " #driver = webdriver.Remote(command_executor='http://localhost:4444', options=chrome_options)\n", " return driver" ] }, @@ -68,16 +76,13 @@ "metadata": {}, "outputs": [], "source": [ - "def saveStats(stats):\n", - "\n", - " with open('stats.json', 'r+') as jsonFile:\n", - " try:\n", - " data = json.load(jsonFile)\n", - " except json.JSONDecodeError:\n", - " data = []\n", - " data.append(stats)\n", - " jsonFile.seek(0)\n", - " json.dump(data, jsonFile, indent=4)\n", + "def saveStats(stats: dict):\n", + " try:\n", + " sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)\n", + " sock.sendto(json.dumps(stats).encode(), ('localhost', 8094))\n", + " sock.close()\n", + " except socket.error as e:\n", + " print(f'Got socket error: {e}')\n", "\n", "def downloadStats(driver: webdriver.Chrome, display_handle: DisplayHandle):\n", " html = driver.find_element(By.CLASS_NAME ,'vjs-stats-list')\n", @@ -103,18 +108,33 @@ " htmlBS.div.insert_before(peersDiv)\n", " \n", " stats = htmlBS.find_all('div', attrs={'style': 'display: block;'})\n", - " statsList = dict(\n", - " map(\n", - " lambda stat: (\n", - " stat.div.text, \n", - " stat.span.text.replace('\\u21d3', 'down').replace('down/', 'down /').replace('\\u21d1 ', 'up').replace('\\u21d1', 'up').replace('\\u00b7', '-').strip()\n", - " ), stats\n", - " )\n", - " )\n", "\n", - " display_handle.update(statsList)\n", + " statsDict = {\n", + " stat.div.text: stat.span.text.replace('\\u21d3', 'down').replace('down/', 'down /').replace('\\u21d1 ', 'up').replace('\\u21d1', 'up').replace('\\u00b7', '-').strip()\n", + " for stat in stats\n", + " }\n", + " \n", + " for stat in statsDict:\n", + " if 'Buffer State' == stat:\n", + " statsDict[stat] = statsDict[stat][1:-1].split(', ')\n", "\n", - " saveStats(statsList)" + " # statsDict = {\n", + " # 'userName': dict(\n", + " # map(\n", + " # lambda stat: (\n", + " # stat.div.text, \n", + " # stat.span.text.replace('\\u21d3', 'down').replace('down/', 'down /').replace('\\u21d1 ', 'up').replace('\\u21d1', 'up').replace('\\u00b7', '-').strip()\n", + " # ), stats\n", + " # )\n", + " # )\n", + " # }\n", + "\n", + " statsDict.update({'Timestamp': time.strftime('%Y-%m-%dT%H:%M:%S%z')})\n", + " statsDict['userName'] = 'user'\n", + "\n", + " display_handle.update(json.dumps(statsDict))\n", + "\n", + " saveStats(statsDict)" ] }, { @@ -134,7 +154,7 @@ " actions.click(driver.find_element(By.CLASS_NAME ,'video-js')).perform()\n", " actions.context_click(driver.find_element(By.CLASS_NAME ,'video-js')).perform()\n", " statsForNerds = driver.find_elements(By.CLASS_NAME ,'vjs-menu-item')\n", - " actions.pause(1)\n", + " actions.pause(2)\n", " actions.click(statsForNerds[-1]).perform()\n", " wait.until(ec.text_to_be_present_in_element((By.CLASS_NAME, 'vjs-stats-list'), 'Player'))\n", " actions.move_to_element(driver.find_element(By.CLASS_NAME ,'vjs-peertube')).perform()\n", @@ -158,11 +178,11 @@ "\n", " signal.signal(signal.SIGINT, lambda signum, frame: interrupt_handler(signum, driver))\n", " \n", - " setupStats(driver, \"https://tube.kobim.cloud/w/gFL48Fz3doCEnYwK46BwYN\")\n", + " setupStats(driver, \"https://tube.kobim.cloud/w/9hAbiwai4rsbw9QnPpPkCd\")\n", "\n", " display_handle = display(\"Loading...\", display_id=True)\n", " \n", - " schedule.every(0.3).seconds.do(downloadStats, driver, display_handle)\n", + " schedule.every(1).seconds.do(downloadStats, driver, display_handle)\n", " while True:\n", " schedule.run_pending()" ] @@ -184,7 +204,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.3" + "version": "3.12.5" } }, "nbformat": 4, diff --git a/peertube/statnerd/telegraf.conf b/peertube/statnerd/telegraf.conf new file mode 100644 index 0000000..e247fda --- /dev/null +++ b/peertube/statnerd/telegraf.conf @@ -0,0 +1,60 @@ +[[processors.dedup]] + dedup_interval = "600s" + +[[outputs.file]] + files = ["stdout"] + +# [[outputs.mongodb]] +# dsn = "mongodb://192.168.86.40:27017" +# database = "peertube" +# granularity = "seconds" +# authentication = "SCRAM" +# username = "root" +# password = "example" + +[[inputs.socket_listener]] + service_address = "udp://:8094" + data_format = "xpath_json" + [[inputs.socket_listener.xpath]] + metric_name = "'stats'" + + [inputs.socket_listener.xpath.fields] + P2P = "/P2P" + #Buffer_State = "string-join(/Buffer_State/*, ',')" + Buffer_State = "/text()='Buffer State'" + Resolution = "/Resolution" + +# [[inputs.socket_listener]] +# service_address = "udp://:8094" +# data_format = "json_v2" +# [[inputs.socket_listener.json_v2]] +# measurement_name = "stats" + +# [[inputs.socket_listener.json_v2.field]] +# path = "P2P" +# [[inputs.socket_listener.json_v2.field]] +# path = "Player mode" +# [[inputs.socket_listener.json_v2.field]] +# path = "P2P" +# [[inputs.socket_listener.json_v2.field]] +# path = "Video UUID" +# [[inputs.socket_listener.json_v2.field]] +# path = "Viewport / Frames" +# [[inputs.socket_listener.json_v2.field]] +# path = "Resolution" +# [[inputs.socket_listener.json_v2.field]] +# path = "Volume" +# [[inputs.socket_listener.json_v2.field]] +# path = "Codecs" +# [[inputs.socket_listener.json_v2.field]] +# path = "Connection Speed" +# [[inputs.socket_listener.json_v2.field]] +# path = "Network Activity" +# [[inputs.socket_listener.json_v2.field]] +# path = "Total Transfered" +# [[inputs.socket_listener.json_v2.field]] +# path = "Download Breakdown" +# [[inputs.socket_listener.json_v2.field]] +# path = "Buffer State" +# [[inputs.socket_listener.json_v2.field]] +# path = "Live Latency" \ No newline at end of file diff --git a/peertube/statnerd/utils/ColoredFormatter.py b/peertube/statnerd/utils/ColoredFormatter.py new file mode 100644 index 0000000..27eb09b --- /dev/null +++ b/peertube/statnerd/utils/ColoredFormatter.py @@ -0,0 +1,43 @@ +import logging + +class ColoredFormatter(logging.Formatter): + """Colored formatter for the logging package.""" + + def __init__( + self, fmt=None, datefmt=None, style="%", validate=True, *, defaults=None + ): + """Colored formatter for the logging package.""" + fmt = fmt or "%(levelname)s: %(message)s" + super().__init__(fmt, datefmt, style, validate, defaults=defaults) + colors = { + "red": "\x1b[31;20m", + "bold_red": "\x1b[31;1m", + "green": "\x1b[32;20m", + "bold_green": "\x1b[32;1m", + "yellow": "\x1b[33;20m", + "bold_yellow": "\x1b[33;1m", + "blue": "\x1b[34;20m", + "bold_blue": "\x1b[34;1m", + "grey": "\x1b[37;20m", + "bold_grey": "\x1b[37;1m", + "reset": "\x1b[0m", + } + self._default_formatter = logging.Formatter(fmt) + self._formatters = { + 100: logging.Formatter(colors["bold_blue"] + fmt + colors["reset"]), + logging.DEBUG: logging.Formatter(colors["grey"] + fmt + colors["reset"]), + logging.INFO: logging.Formatter(colors["green"] + fmt + colors["reset"]), + logging.WARNING: logging.Formatter( + colors["yellow"] + fmt + colors["reset"] + ), + logging.ERROR: logging.Formatter(colors["red"] + fmt + colors["reset"]), + logging.CRITICAL: logging.Formatter( + colors["bold_red"] + fmt + colors["reset"] + ), + } + + def format(self, record): + """Override of logging.Formatter.format""" + return self._formatters.get(record.levelno, self._default_formatter).format( + record + )