feat: update README and main.py to clarify port usage and add error handling for socket port

.env.example

@@ -1,13 +1,8 @@
 # User defined hostname persisted across all sessions, used to keep track of the same user
-# Set to $NODE_NAME to use the hostname of the node when running a cluster with Hetzner Cloud CLI
 TELEGRAF_HOSTNAME=
 # MongoDB connection string
 TELEGRAF_MONGODB_DSN=mongodb://stats_user:%40z%5EVFhN7q%25vzit@tube.kobim.cloud:27107/?authSource=statistics
 # MongoDB database name to store the data
 TELEGRAF_MONGODB_DATABASE=statistics
 # URL of the video to be analyzed
-VIDEO_URL=https://tube.kobim.cloud/w/eT1NZibmwMy6bx6N2YGLwr
-# Selenium Grid Hub URL
-#HUB_URL=http://localhost:4444
-# Socket port to send and listen for incoming data
-#SOCKET_PORT=8094
+VIDEO_URL=https://tube.kobim.cloud/w/iN2T8PmbSb4HJTDA2rV3sg

.gitattributes

@@ -1,2 +0,0 @@
-server/peertube[[:space:]]data/statistics.peertube_hetzner_default_latency.json filter=lfs diff=lfs merge=lfs -text
-server/peertube[[:space:]]data/statistics.peertube_hetzner_high_latency.json filter=lfs diff=lfs merge=lfs -text

.github/workflows/monolith.yml

@@ -7,7 +7,6 @@ on:
 
 env:
   REGISTRY_URL: gitea.kobim.cloud
-  DOCKERHUB_USERNAME: kobimex
 
 jobs:
   build:
@@ -19,37 +18,20 @@ jobs:
       - name: Setup Docker Environment
         uses: ./.github/actions/setup-docker-environment
         
-      - name: Log in to custom Docker registry
+      - name: Log in to Docker registry
         uses: docker/login-action@v3.3.0
         with:
           registry: ${{ env.REGISTRY_URL }}
           username: ${{ github.actor }}
           password: ${{ secrets.REGISTRY_TOKEN }}
           
-      - name: Build and push Docker image to custom registry
+      - name: Build and push Docker image
         uses: docker/build-push-action@v6.13.0
         with:
           context: .
           push: true
           tags: ${{ env.REGISTRY_URL }}/${{ github.repository_owner }}/${{ github.event.repository.name }}-monolith:latest
           file: ./Monolith.dockerfile
-          platforms: |
-            linux/amd64
-            linux/arm64
-
-      - name: Log in to Docker Hub
-        uses: docker/login-action@v3.3.0
-        with:
-          username: ${{ env.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Build and push Docker image to Docker Hub
-        uses: docker/build-push-action@v6.13.0
-        with:
-          context: .
-          push: true
-          tags: ${{ env.DOCKERHUB_USERNAME }}/${{ github.event.repository.name }}-monolith:latest
-          file: ./Monolith.dockerfile
           platforms: |
             linux/amd64
             linux/arm64

.gitignore

@@ -262,7 +262,6 @@ TSWLatexianTemp*
 
 # gummi
 .*.swp
-*.swp
 
 # KBibTeX
 *~[0-9]*
@@ -294,7 +293,6 @@ TSWLatexianTemp*
 .ipynb_checkpoints/
 env/
 .env
-.env.hetzner
 __pycache__/
 test/
 venv/

README.md

@@ -59,13 +59,7 @@ To stop the Docker containers run: `docker compose down -v`
 3. Ajust the firewall settings to allow the necessary ports if needed
 4. Start the Docker container:
     ```sh
-    docker run --rm -p 7900:7900 --env-file .env --name peertube-collector --pull always --shm-size="2g" gitea.kobim.cloud/kobim/peertube-collector-monolith:latest
-    ```
-
-    or
-
-    ```sh
-    docker run --rm -p 7900:7900 --env-file .env --name peertube-collector --pull always --shm-size="2g" kobimex/peertube-collector-monolith:latest
+    docker run --rm -p 7900:7900 --env-file .env --name peertube-collector gitea.kobim.cloud/kobim/peertube-collector-monolith:latest
     ```
 
 

main.py

@@ -1,9 +1,11 @@
 import signal
+import json
 import time
+import socket
 import logging
 import os
 import argparse
-from time import sleep
+from yaspin import yaspin
 from functools import partial
 from http.server import HTTPServer
 from utils.PostHandler import Handler
@@ -17,14 +19,6 @@ from selenium.webdriver import ActionChains
 from selenium.webdriver.support.wait import WebDriverWait
 from selenium.webdriver.support import expected_conditions as ec
 
-# Plugin system imports
-import importlib
-import importlib.util
-import inspect
-import glob
-import sys  # Import the sys module
-from utils.plugins_base import StatsSetupPlugin, StatsDownloadPlugin
-
 logger = logging.getLogger(__name__)
 args = None
 
@@ -47,7 +41,6 @@ def setupArgParser():
     parser.add_argument('--hub-url', type=str, help='URL of the Selenium hub to connect to. If not provided, local Chrome driver will be used.')
     parser.add_argument('--webrtc-internals-path', type=str, help='Path to the WebRTC internals extension.')
     parser.add_argument('--log-level', type=str, help='Log level to use. Default: INFO')
-    parser.add_argument('--plugin-dir', type=str, help='Path to the plugin directory.')
     
     return parser
 
@@ -57,6 +50,7 @@ def interrupt_handler(signum, driver: webdriver.Remote):
     driver.quit()
     raise SystemExit
 
+@yaspin()
 def setupChromeDriver(command_executor: str | None, webrtc_internals_path: str) -> webdriver.Remote | webdriver.Chrome:
     logger.log(logging.INFO, 'Setting up Chrome driver.')
     chrome_options = Options()
@@ -80,215 +74,143 @@ def setupChromeDriver(command_executor: str | None, webrtc_internals_path: str)
     
     return driver
 
-def convert_to_bytes(down, downUnit):
-    return float(down) * (1000 ** {'B': 0, 'KB': 1, 'MB': 2, 'GB': 3}[downUnit])
+def saveStats(stats: list, socket_url: str, socket_port: int):
+    try:
+        sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+        logger.log(logging.DEBUG, f'Saving stats: {json.dumps(stats, indent=4)}')
+        sock.sendto(json.dumps(stats).encode(), (socket_url, socket_port))
+        sock.close()
+        logger.log(logging.DEBUG, 'Sent stats to socket.')
+    except socket.error as e:
+        logger.error(f'Got socket error: {e}')
 
-# Default Plugin Implementations
-class DefaultStatsSetupPlugin(StatsSetupPlugin):
-    def setup_stats(self, driver: webdriver.Remote, url: str, retries: int = 5) -> webdriver.Remote:
-        logger.log(logging.INFO, 'Setting up stats.')
-        actions = ActionChains(driver)
-        wait = WebDriverWait(driver, 30, poll_frequency=0.2)
-        
-        sleep(2)
+def downloadStats(driver: webdriver.Remote | webdriver.Chrome, peersDict: dict, socket_url: str, socket_port: int):
+    html = driver.find_element(By.CLASS_NAME ,'vjs-stats-list').get_attribute('innerHTML')
+    if html is not None:
+        htmlBS = bs(html, 'html.parser')
+    else:
+        raise ValueError("html is None")
+    
+    stats = htmlBS.find_all('div', attrs={'style': 'display: block;'})
 
-        for attempt in range(retries):
-            driver.get(url)
-            try:
-                wait.until(ec.presence_of_element_located((By.CLASS_NAME, 'vjs-big-play-button')))
-                break
-            except Exception:
-                logger.error(f'Timeout while waiting for the big play button to be present. Attempt {attempt + 1} of {retries}')
-                if attempt == retries - 1:
-                    logger.error('Timeout limit reached. Exiting.')
-                    driver.quit()
-                    raise SystemExit(1)
-
-        actions.click(driver.find_element(By.CLASS_NAME ,'video-js')).perform()
-        wait.until(ec.visibility_of_element_located((By.CLASS_NAME, 'vjs-control-bar')))
-        actions.context_click(driver.find_element(By.CLASS_NAME ,'video-js')).perform()
-        statsForNerds = driver.find_elements(By.CLASS_NAME ,'vjs-menu-item')
-        actions.click(statsForNerds[-1]).perform()
-        wait.until(ec.presence_of_element_located((By.CSS_SELECTOR, 'div.vjs-stats-content[style="display: block;"]')))
-        actions.move_to_element(driver.find_element(By.CLASS_NAME ,'vjs-control-bar')).perform()
-        logger.log(logging.INFO, 'Stats setup complete.')
-        
-        return driver
-
-class DefaultStatsDownloadPlugin(StatsDownloadPlugin):
-    def download_stats(self, driver: webdriver.Remote, peersDict: dict, socket_url: str, socket_port: int):
-        html = driver.find_element(By.CLASS_NAME ,'vjs-stats-list').get_attribute('innerHTML')
-        if html is not None:
-            htmlBS = bs(html, 'html.parser')
-        else:
-            raise ValueError("html is None")
-        
-        stats = htmlBS.find_all('div', attrs={'style': 'display: block;'})
-
-        playerStats = {
-            stat.div.text: stat.span.text.replace('\u21d3', 'down').replace('down/', 'down /').replace('\u21d1 ', 'up').replace('\u21d1', 'up').replace('\u00b7', '-').strip() # type: ignore
-            for stat in stats
-        }
-        
-        keys = list(playerStats.keys())
-        for stat in keys:
-            if 'Viewport / Frames' == stat:
-                viewport, frames = playerStats[stat].split(' / ')
-                width, height = viewport.split('x')
-                height, devicePixelRatio = height.split('*')
-                dropped, total = frames.split(' of ')[0].split()[0], frames.split(' of ')[1].split()[0]
-                playerStats[stat] = {'Width': int(width), 'Height': int(height), 'Pixel ratio': float(devicePixelRatio), 'Frames': {'Dropped': int(dropped), 'Total': int(total)}}
-                
-            if 'Codecs' == stat:
-                video, audio = playerStats[stat].split(' / ')
-                playerStats[stat] = {'Video': video, 'Audio': audio}
+    playerStats = {
+        stat.div.text: stat.span.text.replace('\u21d3', 'down').replace('down/', 'down /').replace('\u21d1 ', 'up').replace('\u21d1', 'up').replace('\u00b7', '-').strip()
+        for stat in stats
+    }
+    
+    keys = list(playerStats.keys())
+    for stat in keys:
+        if 'Viewport / Frames' == stat:
+            viewport, frames = playerStats[stat].split(' / ')
+            width, height = viewport.split('x')
+            height, devicePixelRatio = height.split('*')
+            dropped, total = frames.split(' of ')[0].split()[0], frames.split(' of ')[1].split()[0]
+            playerStats[stat] = {'Width': int(width), 'Height': int(height), 'Pixel ratio': float(devicePixelRatio), 'Frames': {'Dropped': int(dropped), 'Total': int(total)}}
             
-            if 'Volume' == stat:
-                if ' (' in playerStats[stat]:
-                    volume, muted = playerStats[stat].split(' (')
-                    playerStats[stat] = {'Volume': int(volume), 'Muted': 'muted' in muted}
-                else:
-                    playerStats[stat] = {'Volume': int(playerStats[stat]), 'Muted': False}
-                
-            if 'Connection Speed' == stat:
-                speed, unit = playerStats[stat].split()
-                
-                speedBytes = float(speed) * (1024 ** {'B/s': 0, 'KB/s': 1, 'MB/s': 2, 'GB/s': 3}[unit])
-                
-                playerStats[stat] = {'Speed': speedBytes, 'Granularity': 's'}
-            
-            if 'Network Activity' == stat:
-                downString, upString = playerStats[stat].split(' / ')
-                
-                down, downUnit = downString.replace('down', '').strip().split()
-                up, upUnit = upString.replace('up', '').strip().split()
-                
-                downBytes = convert_to_bytes(down, downUnit)
-                upBytes = convert_to_bytes(up, upUnit)
-                
-                playerStats[stat] = {'Down': downBytes, 'Up': upBytes}
-                
-            if 'Total Transfered' == stat:
-                downString, upString = playerStats[stat].split(' / ')
-                
-                down, downUnit = downString.replace('down', '').strip().split()
-                up, upUnit = upString.replace('up', '').strip().split()
-                
-                downBytes = convert_to_bytes(down, downUnit)
-                upBytes = convert_to_bytes(up, upUnit)
-                
-                playerStats[stat] = {'Down': downBytes, 'Up': upBytes}
-                
-            if 'Download Breakdown' == stat:
-                server, peer = playerStats[stat].split(' - ')
-                
-                server, serverUnit = server.replace('from servers', '').strip().split()
-                peer, peerUnit = peer.replace('from peers', '').strip().split()
-                
-                serverBytes = convert_to_bytes(server, serverUnit)
-                peerBytes = convert_to_bytes(peer, peerUnit)
-                
-                playerStats[stat] = {'Server': serverBytes, 'Peers': peerBytes}
-                
-            if 'Buffer State' == stat:
-                del(playerStats[stat])
-                
-            if 'Live Latency' == stat:
-                latency, edge = playerStats[stat].split(' (from edge: ')
-                latency = sum(int(x) * 60 ** i for i, x in enumerate(reversed([part for part in latency.replace('s', '').split('m') if part])))
-                edge = sum(int(x) * 60 ** i for i, x in enumerate(reversed([part for part in edge.replace('s', '').replace(')', '').split('m') if part])))
-                playerStats[stat] = {'Latency': latency, 'Edge': edge}
+        if 'Codecs' == stat:
+            video, audio = playerStats[stat].split(' / ')
+            playerStats[stat] = {'Video': video, 'Audio': audio}
         
-        stats = {
-            'player': playerStats,
-            'peers': peersDict,
-            'url': driver.current_url,
-            'timestamp': int(time.time() * 1000),
-            'session': driver.session_id
-        }
-
-        super().saveStats([stats], socket_url, socket_port)
-
-# Plugin loading mechanism
-def load_plugins(plugin_dir: str) -> tuple[StatsSetupPlugin | None, StatsDownloadPlugin | None]:
-    """
-    Loads plugins from the specified directory.
-
-    Args:
-        plugin_dir: The directory to search for plugins.
-
-    Returns:
-        A tuple containing the loaded StatsSetupPlugin and StatsDownloadPlugin, or (None, None) if no plugins were found.
-    """
-    
-    logger.info(f"Loading plugins from {plugin_dir}")
-    
-    setup_plugin = None
-    download_plugin = None
-
-    plugin_files = glob.glob(os.path.join(plugin_dir, "*.py"))
-    
-    # Log the contents of the plugin directory
-    logger.debug(f"Plugin directory contents: {os.listdir(plugin_dir)}")
-    
-    for plugin_file in plugin_files:
-        module_name = os.path.basename(plugin_file)[:-3]  # Remove .py extension
-        logger.debug(f"Loading plugin file {plugin_file}")
-        try:
-            spec = importlib.util.spec_from_file_location(module_name, plugin_file)
-            logger.debug(f"Spec: {spec}")
-            if spec is None:
-                logger.warning(f"Can't load plugin file {plugin_file}")
-                continue
-            module = importlib.util.module_from_spec(spec)
-            logger.debug(f"Module: {module}")
-            if spec.loader is not None:
-                spec.loader.exec_module(module)
+        if 'Volume' == stat:
+            if ' (' in playerStats[stat]:
+                volume, muted = playerStats[stat].split(' (')
+                playerStats[stat] = {'Volume': int(volume), 'Muted': 'muted' in muted}
             else:
-                logger.warning(f"Can't load module {module_name} from {plugin_file}")
+                playerStats[stat] = {'Volume': int(playerStats[stat]), 'Muted': False}
             
-            for name, obj in inspect.getmembers(module):
-                logger.debug(f"Found member: {name} in module {module_name}")
-                if inspect.isclass(obj):
-                    if issubclass(obj, StatsSetupPlugin) and obj is not StatsSetupPlugin:
-                        logger.info(f"Found StatsSetupPlugin: {obj.__name__}")
-                        setup_plugin = obj()
-                        logger.debug(f"Loaded StatsSetupPlugin: {obj.__name__} from {plugin_file}")
-                    elif issubclass(obj, StatsDownloadPlugin) and obj is not StatsDownloadPlugin:
-                        logger.info(f"Found StatsDownloadPlugin: {obj.__name__}")
-                        download_plugin = obj()
-                        logger.debug(f"Loaded StatsDownloadPlugin: {obj.__name__} from {plugin_file}")
-                    else:
-                        logger.debug(f"Class {obj.__name__} is not a subclass of StatsSetupPlugin or StatsDownloadPlugin")
-                else:
-                    logger.debug(f"{name} is not a class")
-        except Exception as e:
-            logger.warning(f"Error loading plugin {plugin_file}: {e}")
+        if 'Connection Speed' == stat:
+            speed, unit = playerStats[stat].split()
+            
+            speedBytes = float(speed) * (1024 ** {'B/s': 0, 'KB/s': 1, 'MB/s': 2, 'GB/s': 3}[unit])
+            
+            playerStats[stat] = {'Speed': speedBytes, 'Granularity': 's'}
+        
+        if 'Network Activity' == stat:
+            downString, upString = playerStats[stat].split(' / ')
+            
+            down, downUnit = downString.replace('down', '').strip().split()
+            up, upUnit = upString.replace('up', '').strip().split()
+            
+            downBytes = convert_to_bytes(down, downUnit)
+            upBytes = convert_to_bytes(up, upUnit)
+            
+            playerStats[stat] = {'Down': downBytes, 'Up': upBytes}
+            
+        if 'Total Transfered' == stat:
+            downString, upString = playerStats[stat].split(' / ')
+            
+            down, downUnit = downString.replace('down', '').strip().split()
+            up, upUnit = upString.replace('up', '').strip().split()
+            
+            downBytes = convert_to_bytes(down, downUnit)
+            upBytes = convert_to_bytes(up, upUnit)
+            
+            playerStats[stat] = {'Down': downBytes, 'Up': upBytes}
+            
+        if 'Download Breakdown' == stat:
+            server, peer = playerStats[stat].split(' - ')
+            
+            server, serverUnit = server.replace('from servers', '').strip().split()
+            peer, peerUnit = peer.replace('from peers', '').strip().split()
+            
+            serverBytes = convert_to_bytes(server, serverUnit)
+            peerBytes = convert_to_bytes(peer, peerUnit)
+            
+            playerStats[stat] = {'Server': serverBytes, 'Peers': peerBytes}
+            
+        if 'Buffer State' == stat:
+            del(playerStats[stat])
+            
+        if 'Live Latency' == stat:
+            latency, edge = playerStats[stat].split(' (from edge: ')
+            latency = sum(int(x) * 60 ** i for i, x in enumerate(reversed([part for part in latency.replace('s', '').split('m') if part])))
+            edge = sum(int(x) * 60 ** i for i, x in enumerate(reversed([part for part in edge.replace('s', '').replace(')', '').split('m') if part])))
+            playerStats[stat] = {'Latency': latency, 'Edge': edge}
+    
+    stats = {
+        'player': playerStats,
+        'peers': peersDict,
+        'url': driver.current_url,
+        'timestamp': int(time.time() * 1000),
+        'session': driver.session_id
+    }
 
-    return setup_plugin, download_plugin
+    saveStats([stats], socket_url, socket_port)
+
+def convert_to_bytes(down, downUnit):
+    return float(down) * (1024 ** {'B': 0, 'KB': 1, 'MB': 2, 'GB': 3}[downUnit])
+
+@yaspin()
+def setupStats(driver: webdriver.Remote, url: str):
+    logger.log(logging.INFO, 'Setting up stats.')
+    actions = ActionChains(driver)
+    wait = WebDriverWait(driver, 30, poll_frequency=0.2)
+
+    driver.get(url)
+
+    try:
+        wait.until(ec.presence_of_element_located((By.CLASS_NAME, 'vjs-big-play-button')))
+    except Exception:
+        logger.error('Timeout while waiting for the big play button to be present.')
+        driver.quit()
+        raise SystemExit(1)
+
+    actions.click(driver.find_element(By.CLASS_NAME ,'video-js')).perform()
+    wait.until(ec.visibility_of_element_located((By.CLASS_NAME, 'vjs-control-bar')))
+    actions.context_click(driver.find_element(By.CLASS_NAME ,'video-js')).perform()
+    statsForNerds = driver.find_elements(By.CLASS_NAME ,'vjs-menu-item')
+    actions.click(statsForNerds[-1]).perform()
+    wait.until(ec.text_to_be_present_in_element((By.CLASS_NAME, 'vjs-stats-list'), 'Player'))
+    actions.move_to_element(driver.find_element(By.CLASS_NAME ,'vjs-control-bar')).perform()
+    logger.log(logging.INFO, 'Stats setup complete.')
+    
+    return driver
 
 if __name__ == '__main__':
     args = setupArgParser().parse_args()
     
     setupLogger()
     
-    # Load plugins
-    plugin_dir = firstValid(args.plugin_dir, os.getenv('PLUGIN_DIR'), default=None)
-    if plugin_dir is None:
-        logger.info("No plugin directory provided. Using default plugins.")
-        setup_plugin = None
-        download_plugin = None
-    else:
-        setup_plugin, download_plugin = load_plugins(plugin_dir)
-
-    # Use default plugins if none are loaded
-    if setup_plugin is None:
-        setup_plugin = DefaultStatsSetupPlugin()
-        logger.info("Using default StatsSetupPlugin.")
-    if download_plugin is None:
-        download_plugin = DefaultStatsDownloadPlugin()
-        logger.info("Using default StatsDownloadPlugin.")
-    
     command_executor = firstValid(args.hub_url, os.getenv('HUB_URL'), default=None)
     webrtc_internals_path = firstValid(
         args.webrtc_internals_path,
@@ -305,8 +227,7 @@ if __name__ == '__main__':
         logger.error('VIDEO_URL environment variable or --url argument is required.')
         raise SystemExit(1)
     
-    # Use the loaded plugin
-    driver = setup_plugin.setup_stats(driver, url)
+    setupStats(driver, url)
     
     socket_url = firstValid(args.socket_url, os.getenv('SOCKET_URL'), default='localhost')
     try:
@@ -316,5 +237,5 @@ if __name__ == '__main__':
         raise SystemExit(1)
     
     logger.info('Starting server collector.')
-    httpd = HTTPServer(('', 9092), partial(Handler, download_plugin.download_stats, driver, logger, socket_url, socket_port))
+    httpd = HTTPServer(('', 9092), partial(Handler, downloadStats, driver, logger, socket_url, socket_port))
     httpd.serve_forever()

monolith-entrypoint.sh

@@ -48,8 +48,6 @@ done
 
 printf '\n'
 
-# Start the Telegraf agent
+# Start the Telegraf agent and the main script
 telegraf --config ./telegraf.conf &
-
-# Start the main Python script as PID 1
-exec ./venv/bin/python main.py
+./venv/bin/python main.py

plugins/example_plugin.py

@@ -1,31 +0,0 @@
-import logging
-from selenium import webdriver
-from selenium.webdriver.remote.webdriver import WebDriver as Remote
-from utils.plugins_base import StatsSetupPlugin, StatsDownloadPlugin
-
-logger = logging.getLogger(__name__)
-
-class ExampleStatsSetupPlugin(StatsSetupPlugin):
-    def setup_stats(self, driver: webdriver.Chrome, url: str, retries: int = 5) -> webdriver.Chrome:
-        logger.info("Running ExampleStatsSetupPlugin...")
-        # Here you would implement the custom logic to setup stats
-        # For example, you could click on a button to display stats.
-        # You could also wait for an element to appear before continuing.
-        # This is just an example
-        
-        driver.get(url)
-
-        return driver
-
-class ExampleStatsDownloadPlugin(StatsDownloadPlugin):
-    def download_stats(self, driver: webdriver.Chrome, peersDict: dict, socket_url: str, socket_port: int):
-        logger.info("Running ExampleStatsDownloadPlugin...")
-        stats = {'message': 'Hello from ExampleStatsDownloadPlugin'}
-        # Here you would implement the custom logic to download stats
-        # and send them to the socket.
-        # This is just an example
-        
-        print(f"Sending stats: {stats} to {socket_url}:{socket_port}")
-        
-        # Remember to call the saveStats method to send the stats to the socket
-        super().saveStats([stats], socket_url, socket_port)

selenium-standalone-stack/README.md

@@ -1,29 +0,0 @@
-# Selenium standalone grid deployment script
-
-## Cloud provider
-
-This script use the services of Hetzner.
-
-It should be easily modified to use other cloud providers.
-
-## Dependencies
-
-You need to install `jq`, `nmap` and `hcloud`, the Hetzner cloud API CLI.
-
-On Debian
-```bash
-apt install jq nmap hcloud-cli
-```
-
-## Usage
-
-Just read the help provided by the script
-
-```bash
-./create-selenium-stack.sh -h
-```
-
-To remove all servers in the context:
-```bash
-./create-selenium-stack.sh -d -y
-```

selenium-standalone-stack/create-selenium-stack.sh

@@ -1,288 +0,0 @@
-#!/bin/bash
-
-set -m # Enable Job Control
-
-trap 'kill $(jobs -p)' SIGINT
-
-# Reset
-NC='\033[0m'              # Text Reset
-
-# Regular Colors
-Red='\033[0;31m'          # Red
-Green='\033[0;32m'        # Green
-Cyan='\033[0;36m'         # Cyan
-
-if [[ -z $(which hcloud) ]]; then
-    echo -e "${Red}hcloud could not be found in \$PATH!${NC}
-
-Please put hcloud in \$PATH ($PATH),
-install it with your package manager
-or go to https://github.com/hetznercloud/cli/releases to download it."
-    exit 1
-fi
-
-if [[ -z $(which jq) ]]; then
-    echo -e "${Red}jq could not be found in \$PATH!${NC}
-
-Please install jq to use this script."
-    exit 1
-fi
-
-if [[ -z $(which nmap) ]]; then
-    echo -e "${Red}nmap could not be found in \$PATH!${NC}
-
-Please install nmap to use this script."
-    exit 1
-fi
-
-usage() {
-    if hcloud context list | grep -q -v "ACTIVE"; then
-        types=$(hcloud server-type list -o columns=name,cores,cpu_type,memory,storage_type,architecture | grep -v arm | sed -e 's/^/    /')
-        keys=$(hcloud ssh-key list -o columns=name,fingerprint,age | sed -e 's/^/    /')
-        contexts="    Available contexts:
-$(hcloud context list | sed -e 's/^/    /')"
-    else
-        types="No hcloud context, can’t get server types"
-        keys="No hcloud context, can’t get SSH keys"
-        contexts="No hcloud context available.
-You can create one with the following command:
-    hcloud create context name_of_the_context
-Or let this script create one during execution."
-    fi
-
-    cat << EOF
-$(basename "$0") (c) Framasoft 2023, WTFPL
-
-USAGE
-    $(basename "$0") [-h] [-d] [-s <int>] [-n <int>] [-t <vps type>] [-c <hcloud context>] -k <ssh-key>
-
-OPTIONS
-    -h                  Print this help and exit
-    -d                  Delete all servers
-    -dy                 Delete all servers without confirmation
-    -s <int>            How many VPS you want to start.
-                        Default: 1
-                        Maximum should be: limit (hcloud).
-                        Default: 1
-    -n <int>            How many nodes you want to start on each VPS.
-                        Default: 1
-    -t <vps type>       The type of VPS to start.
-                        Default: cpx21.
-                        See below
-    -c <hcloud context> Name of the hcloud context
-                        Default: selenium-peertube.
-                        See below
-    -k <ssh-key>        The ssh key used to connect to the VPS.
-                        MANDATORY, no default.Starting node
-                        See below.
-    -e <string>         The path to the environment file to be copied and used on the VPS.
-                        Default: .env
-
-$types
-
-HCLOUD CONTEXT
-    It’s the name of the project you want to create your VPS in.
-
-$contexts
-
-SSH KEYS
-    You must have a ssh key registered on Hetzner to use this script.
-    To create a key:
-        hcloud ssh-key create --name my-key --public-key-from-file ~/.ssh/id_ed25519.pub
-
-    The ssh keys currently registered on Hetzner are:
-$keys
-EOF
-    exit "$1"
-}
-
-delete_server() {
-    echo -e "${Cyan}$(hcloud server delete "$1")${NC}"
-}
-
-create_nodes_server() {
-    i="$1"
-    TYPE="$2"
-    KEY="$3"
-    REGION="$4"
-    SERVER_NAME="$REGION-node-$i"
-    hcloud server create --start-after-create --name "$SERVER_NAME" --image debian-12 --type "$TYPE" --location "$REGION" --ssh-key "$KEY" > /dev/null
-    echo -e "${Cyan}VPS n°$i created and started${NC}"
-}
-
-start_nodes() {
-    i="$1"
-    REGION=$(hcloud server list -o json | jq -r '.[] | select(.name | contains("node-'$i'")) | .datacenter.location.name')
-    SERVER_NAME="$REGION-node-$i"
-    SERVER_IP=$(hcloud server ip "$SERVER_NAME")
-    while [[ $(nmap -p 22 "$SERVER_IP" | grep -c open) -eq 0 ]]; do
-        sleep 1
-    done
-    SSH_CONN="root@$SERVER_IP"
-    scp -o "LogLevel=ERROR" -o "UserKnownHostsFile /dev/null" -o "StrictHostKeyChecking no" -o "VerifyHostKeyDNS no" start-nodes.sh "${SSH_CONN}:" > /dev/null
-    scp -o "LogLevel=ERROR" -o "UserKnownHostsFile /dev/null" -o "StrictHostKeyChecking no" -o "VerifyHostKeyDNS no" "$ENV_FILE" "${SSH_CONN}:" > /dev/null
-    ssh -o "LogLevel=ERROR" -o "UserKnownHostsFile /dev/null" -o "StrictHostKeyChecking no" -o "VerifyHostKeyDNS no" "$SSH_CONN" "/root/start-nodes.sh -n \"$NODES\"" > /dev/null
-    echo -e "${Cyan}Nodes created on VPS n°${i}${NC}"
-}
-
-CONTEXT=selenium-peertube
-SERVERS=1
-NODES=1
-TYPE=cpx21
-DELETE=0
-N_STRING=node
-FORCE_DELETION=0
-ENV_FILE=.env
-
-while getopts "hds:n:t:k:c:y" option; do
-    case $option in
-        h)
-            usage 0
-            ;;
-        d)
-            DELETE=1
-            ;;
-        s)
-            SERVERS=$OPTARG
-            ;;
-        n)
-            NODES=$OPTARG
-            if [[ $NODES -gt 1 ]]; then
-                N_STRING=nodes
-            fi
-            ;;
-        t)
-            TYPE=$OPTARG
-            ;;
-        k)
-            KEY=$OPTARG
-            ;;
-        c)
-            CONTEXT=$OPTARG
-            ;;
-        y)
-            FORCE_DELETION=1
-            ;;
-        *)
-            usage 1
-            ;;
-    esac
-done
-
-if [[ $(hcloud context active) != "$CONTEXT" ]]; then
-    echo -e "${Cyan}Hcloud context is not '$CONTEXT'!${NC}"
-    if hcloud context list | grep -q -F "$CONTEXT"; then
-        echo -e "${Green}Selecting hcloud context ${CONTEXT}${NC}"
-        hcloud context use "$CONTEXT"
-    else
-        echo -e "${Red}Hcloud context ${CONTEXT} does not exist.${NC}
-${Cyan}Will now try to create the context ${CONTEXT}${NC}"
-        hcloud context create "$CONTEXT"
-    fi
-
-    exit 1
-fi
-
-if [[ $DELETE -eq 1 ]]; then
-    SERVERS=$(hcloud server list -o json)
-    if [[ $SERVERS == 'null' ]]; then
-        echo -e "${Cyan}No VPS to delete.${NC}"
-        exit 0
-    fi
-    NAMES=$(echo "$SERVERS" | jq -r '.[] | .name' | sort -h)
-    echo -e "${Red}You are about to delete the following VPS${NC}:"
-    echo "$NAMES"
-    if [[ $FORCE_DELETION -eq 1 ]]; then
-        confirm="yes"
-    else
-        echo -e -n "${Cyan}Please confirm the deletion by typing '${NC}${Red}yes${NC}': "
-        read -r confirm
-    fi
-    if [[ $confirm == 'yes' ]]; then
-        for i in $NAMES; do
-            echo -e "${Cyan}Starting server $i deletion${NC}"
-            delete_server "$i" &
-        done
-        # Wait for all delete_server jobs to finish
-        while true; do
-            fg > /dev/null 2>&1
-            [ $? == 1 ] && break
-        done
-        if [[ $(hcloud server list -o json) == '[]' ]]; then
-            echo -e "${Green}All servers have been deleted${NC}"
-        else
-            echo -e "${Red}Some servers have not been deleted:${NC}"
-            hcloud server list
-        fi
-    else
-        echo "Deletion cancelled."
-    fi
-    exit 0
-fi
-
-if [[ -z $KEY ]]; then
-    echo -e "${Red}You must choose a ssh key!${NC}\n"
-    usage 1
-fi
-
-KEY_FOUND=0
-for i in $(hcloud ssh-key list -o json | jq -r '.[] | .name'); do
-    if [[ $i == "$KEY" ]]; then
-        KEY_FOUND=1
-        break
-    fi
-done
-
-if [[ $KEY_FOUND -eq 0 ]]; then
-    echo -e "${Red}The chosen ssh key is not registered on Hetzner!${NC}\n"
-    usage 1
-fi
-
-if hcloud server list | grep -q -v NAME; then
-    echo -e "${Red}There already are servers in the context! Exiting.${NC}\nList of the servers:"
-    hcloud server list
-    exit 1
-fi
-
-if [[ ! -f "$ENV_FILE" ]]; then
-    echo -e "${Red}Environment file '$ENV_FILE' does not exist!${NC}"
-    exit 1
-fi
-
-echo -e "${Green}Creating $SERVERS VPS${NC}"
-REGIONS=($(hcloud location list -o json | jq -r '.[] | select(.name != "fsn1") | .name' | shuf))
-for i in $(seq 1 "$SERVERS"); do
-    REGION=${REGIONS[$((i % ${#REGIONS[@]}))]}
-    echo -e "${Cyan}Creating VPS n°$i in $REGION"
-    create_nodes_server "$i" "$TYPE" "$KEY" "$REGION" &
-done
-
-# Wait for all create_nodes_server jobs to finish
-while true; do
-    fg > /dev/null 2>&1
-    [ $? == 1 ] && break
-done
-
-echo -e "${Green}Starting nodes on $SERVERS VPS ($NODES $N_STRING each)${NC}"
-for i in $(seq 1 "$SERVERS"); do
-    echo -e "${Cyan}Starting $N_STRING on VPS n°$i${NC}"
-    start_nodes "$i" &
-done
-
-echo -e "${Green}Waiting for all nodes to be started${NC}"
-
-# Wait for all start_nodes jobs to finish
-while true; do
-    fg > /dev/null 2>&1
-    [ $? == 1 ] && break
-done
-
-echo -e "${Green}All the servers and nodes have been created and started!
-
-Number of servers: $SERVERS
-Number of nodes per server: $NODES
-Type of the servers:
-  nodes servers: $TYPE
-
-You can remove all servers with the following command
-    $0 -d${NC}"

selenium-standalone-stack/start-nodes.sh

@@ -1,126 +0,0 @@
-#!/bin/bash
-
-usage() {
-    cat << EOF
-$(basename "$0") (c) Framasoft 2023, WTPF
-
-USAGE
-    $(basename "$0") [-h] [-n <int>]
-
-OPTIONS
-    -h               print this help and exit
-    -n <int>         how many selenium nodes you want to launch. Default: 1
-    -e <string>      the environment file path to use. Default: .env
-EOF
-    exit "$1"
-}
-
-NUMBER=1
-ENV_FILE=".env"
-
-while getopts "hn:i:" option; do
-    case $option in
-        h)
-            usage 0
-            ;;
-        n)
-            NUMBER=$OPTARG
-            ;;
-        e)
-            ENV_FILE=$OPTARG
-            ;;
-        *)
-            usage 1
-            ;;
-    esac
-done
-
-HOST=$(hostname)
-
-DEBIAN_FRONTEND=noninteractive
-export DEBIAN_FRONTEND
-
-echo "Installing packages"
-apt-get -qq -y update
-apt-get -qq -y dist-upgrade
-apt-get -qq -y install jq \
-                       tmux \
-                       vim \
-                       multitail \
-                       htop \
-                       liquidprompt \
-                       coreutils \
-                       apparmor-utils \
-                       docker.io \
-
-echo "Activating liquidprompt"
-liquidprompt_activate
-. /usr/share/liquidprompt/liquidprompt
-
-echo "Modifying kernel parameters"
-sysctl net.ipv6.conf.default.forwarding=1
-sysctl net.ipv6.conf.all.forwarding=1
-
-echo "Configuring Docker for IPv6"
-IP_ADDR=$(ip --json a show eth0 | jq '.[] | .addr_info | .[] | select(.family | contains("inet6")) | select(.scope | contains("global")) | .local' -r)
-NETWORK=$(echo "$IP_ADDR" | sed -e 's@:[^:]\+$@8000::/65@')
-
-cat << EOF > /etc/docker/daemon.json
-{
-  "ipv6": true,
-  "fixed-cidr-v6": "$NETWORK"
-}
-EOF
-systemctl restart docker
-
-echo "Starting $NUMBER Selenium nodes"
-
-for NB in $(seq 1 "$NUMBER"); do
-    NODE_NAME="selenium-${HOST}-instance-${NB}"
-
-    # Replace variables in the environment file
-    TEMP_ENV_FILE=$(mktemp)
-    while IFS= read -r line; do
-        eval "echo \"$line\""
-    done < "$ENV_FILE" > "$TEMP_ENV_FILE"
-    ENV_FILE="$TEMP_ENV_FILE"
-
-    echo "Starting Selenium node n°$NB"
-    docker run --rm \
-               --env-file $ENV_FILE \
-               --name "$NODE_NAME" \
-               --pull always \
-               --shm-size="2g" \
-               -d \
-               kobimex/peertube-collector-monolith:latest > /dev/null 2>&1
-
-    # Wait until the container gets an IPv6 address.
-    DOCKER_IP=""
-    for i in {1..10}; do
-        DOCKER_IP=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.GlobalIPv6Address}}{{end}}' "$NODE_NAME")
-        if [ -n "$DOCKER_IP" ]; then
-            break
-        fi
-        sleep 1
-    done
-
-    if [ -z "$DOCKER_IP" ]; then
-        echo "Error: Could not retrieve a valid IPv6 address for $NODE_NAME." >&2
-        exit 1
-    fi
-
-    echo "Adding Selenium node n°$NB to neighbour proxy"
-    ip -6 neighbour add proxy "$DOCKER_IP" dev eth0
-    docker stop "$NODE_NAME"
-    sleep 1
-
-    docker run --rm \
-               --env-file $ENV_FILE \
-               --name "$NODE_NAME" \
-               --pull always \
-               --shm-size="2g" \
-               -d \
-               -p 790$NB:790$NB \
-               -e "SE_NO_VNC_PORT=790$NB" \
-               kobimex/peertube-collector-monolith:latest > /dev/null 2>&1
-done

utils/plugins_base.py

@@ -1,29 +0,0 @@
-import abc
-import json
-import socket
-import logging
-from selenium import webdriver
-
-logger = logging.getLogger(__name__)
-
-# Abstract Base Classes for Plugins
-class StatsSetupPlugin(abc.ABC):
-    @abc.abstractmethod
-    def setup_stats(self, driver: webdriver.Remote | webdriver.Chrome, url: str, retries: int = 5) -> webdriver.Remote | webdriver.Chrome:
-        pass
-
-class StatsDownloadPlugin(abc.ABC):
-    @abc.abstractmethod
-    def download_stats(self, driver: webdriver.Remote | webdriver.Chrome, peersDict: dict, socket_url: str, socket_port: int):
-        pass
-
-    @staticmethod
-    def saveStats(stats: list, socket_url: str, socket_port: int):
-        try:
-            sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
-            logger.debug(f'Saving stats: {json.dumps(stats, indent=4)}')
-            sock.sendto(json.dumps(stats).encode(), (socket_url, socket_port))
-            sock.close()
-            logger.debug('Sent stats to socket.')
-        except socket.error as e:
-            logger.error(f'Got socket error: {e}')