feat: add convenience functions, update Docker setup, and integrate Webpack for building WebRTC internals exporter

.github/workflows/feature-pr-build.yml

@@ -1,4 +1,4 @@
-name: Build Docker Image for Pull Request
+name: Build Docker Images for Pull Request
 
 on:
   pull_request:
@@ -25,3 +25,13 @@ jobs:
           platforms: |
             linux/amd64
             linux/arm64
+      
+      - name: Build monolith Docker image
+        uses: docker/build-push-action@v6.13.0
+        with:
+          context: .
+          tags: ${{ env.REGISTRY_URL }}/${{ github.event.repository.name }}:${{ github.event.pull_request.number }}-monolith
+          file: ./Monolith.Dockerfile
+          platforms: |
+            linux/amd64
+            linux/arm64

.github/workflows/monolith.yml

@@ -0,0 +1,37 @@
+name: Build and Push Docker Image
+
+on:
+  push:
+    branches:
+      - main
+
+env:
+  REGISTRY_URL: gitea.kobim.cloud
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository actions
+        uses: actions/checkout@v2
+        
+      - name: Setup Docker Environment
+        uses: ./.github/actions/setup-docker-environment
+        
+      - name: Log in to Docker registry
+        uses: docker/login-action@v3.3.0
+        with:
+          registry: ${{ env.REGISTRY_URL }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.REGISTRY_TOKEN }}
+          
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v6.13.0
+        with:
+          context: .
+          push: true
+          tags: ${{ env.REGISTRY_URL }}/${{ github.repository_owner }}/${{ github.event.repository.name }}-monolith:latest
+          file: ./Monolith.Dockerfile
+          platforms: |
+            linux/amd64
+            linux/arm64

.gitignore

@@ -296,4 +296,11 @@ env/
 __pycache__/
 test/
 venv/
-.venv/
+.venv/
+
+# Node.js
+node_modules/
+npm-debug.log
+yarn-error.log
+yarn-debug.log*
+background.bundle.js

Dockerfile

@@ -1,19 +1,15 @@
 FROM debian:bookworm-slim
 
-# Install Python
-RUN apt-get update && apt-get install -y python3 python3-pip python3-venv
+# Install Python and curl
+RUN apt-get update && apt-get install -y python3 python3-pip python3-venv curl
 
-# Install dependencies with venv
 # Create and activate a virtual environment
 RUN python3 -m venv /app/venv
 ENV PATH="/app/venv/bin:$PATH"
 
 # Install dependencies with venv
 COPY requirements.txt /app/requirements.txt
-RUN /app/venv/bin/pip install --no-cache-dir -r /app/requirements.txt
-
-# Install curl
-RUN apt-get update && apt-get install -y curl
+RUN /app/venv/bin/pip install -r /app/requirements.txt
 
 # Copy the application
 COPY main.py /app
@@ -24,4 +20,4 @@ WORKDIR /app
 CMD ["/app/venv/bin/python", "main.py"]
 
 # Healthcheck
-HEALTHCHECK --interval=5s --timeout=10s --retries=5 CMD curl -f http://localhost/healthcheck || exit 1
+HEALTHCHECK --interval=5s --timeout=10s --retries=5 --start-period=5s CMD curl -f http://localhost:9092/heartbeat || exit 1

Monolith.dockerfile

@@ -0,0 +1,52 @@
+FROM node:22.14.0-bookworm-slim AS build
+
+# Copy the webrtc-internals-exporter files
+COPY webrtc-internals-exporter /tmp/webrtc-internals-exporter
+
+WORKDIR /tmp/webrtc-internals-exporter/webpack
+
+# Install dependencies
+RUN --mount=type=cache,target=/root/.npm \
+    npm install
+
+# Build the project
+RUN npm run build
+
+FROM selenium/standalone-chromium:129.0
+
+# Install Python-virtualenv
+RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/apt \
+    sudo apt-get update && sudo sudo apt-get install -y python3-venv
+
+WORKDIR /tmp
+
+# Install Telegraf
+RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/apt \
+    wget -q https://repos.influxdata.com/influxdata-archive_compat.key && \
+    echo '393e8779c89ac8d958f81f942f9ad7fb82a25e133faddaf92e15b16e6ac9ce4c influxdata-archive_compat.key' | sha256sum -c && \
+    cat influxdata-archive_compat.key | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/influxdata-archive_compat.gpg > /dev/null && \
+    echo 'deb [signed-by=/etc/apt/trusted.gpg.d/influxdata-archive_compat.gpg] https://repos.influxdata.com/debian stable main' | sudo tee /etc/apt/sources.list.d/influxdata.list && \
+    sudo apt-get update && sudo apt-get install -y telegraf
+
+# Create and activate a virtual environment
+RUN python3 -m venv ./venv
+ENV PATH="/tmp/venv/bin:$PATH"
+
+# Install dependencies with venv
+COPY requirements.txt ./requirements.txt
+RUN ./venv/bin/pip install -r ./requirements.txt
+
+# Copy the application
+COPY main.py .
+COPY utils/ ./utils
+COPY telegraf.conf ./telegraf.conf
+COPY webrtc-internals-exporter /tmp/webrtc-internals-exporter
+COPY --from=build /tmp/webrtc-internals-exporter/background.bundle.js /tmp/webrtc-internals-exporter/background.bundle.js
+COPY --chown="${SEL_UID}:${SEL_GID}" monolith-entrypoint.sh /opt/bin/collector.sh
+
+# Run the entrypoint
+RUN chmod +x /opt/bin/collector.sh
+ENTRYPOINT ["/opt/bin/collector.sh"]
+
+# Healthcheck
+HEALTHCHECK --interval=5s --timeout=10s --retries=5 --start-period=5s CMD curl -f http://localhost:9092/heartbeat || exit 1

docker-compose.yml

@@ -3,7 +3,7 @@ services:
     container_name: selenium-standalone-chromium
     image: selenium/standalone-chromium:129.0
     volumes:
-      - ./webrtc-internals-exporter:/tmp/webrtc-internals-exporter:ro
+      - build-extension:/tmp/webrtc-internals-exporter
     shm_size: "2g"
     attach: false
     depends_on:
@@ -15,7 +15,10 @@ services:
       timeout: 10s
       retries: 5
     pull_policy: always
-    network_mode: host
+    ports:
+      - "7900:7900"
+    networks:
+      - backend
 
   telegraf:
     container_name: telegraf
@@ -34,6 +37,25 @@ services:
     pull_policy: always
     networks:
       - backend
+  build-extension:
+    container_name: build-extension
+    image: node:22.14.0-bookworm-slim
+    volumes:
+      - ./webrtc-internals-exporter:/tmp/webrtc-internals-exporter:ro
+      - build-extension:/tmp/webrtc-internals-exporter-build
+    working_dir: /tmp/webrtc-internals-exporter-build/webpack
+    command:
+      - /bin/bash
+      - -c
+      - |
+        cp -r /tmp/webrtc-internals-exporter/* /tmp/webrtc-internals-exporter-build
+        npm install
+        npm run build
+    environment:
+      - WEBRTC_INTERNALS_EXPORTER_URL=http://collector
+    pull_policy: always
+    networks:
+      - backend
 
   collector:
     container_name: collector
@@ -46,12 +68,13 @@ services:
         condition: service_healthy
       telegraf:
         condition: service_healthy
+      build-extension:
+        condition: service_completed_successfully
     environment:
       - VIDEO_URL=${VIDEO_URL:?"Video URL is required"}
-    ports:
-      - "9092:9092"
-    extra_hosts:
-      - "host.docker.internal:host-gateway"
+      - SOCKET_URL=telegraf
+      - HUB_URL=http://selenium:4444
+      - WEBRTC_INTERNALS_PATH=/tmp/webrtc-internals-exporter
     pull_policy: always
     networks:
       - backend
@@ -61,3 +84,6 @@ networks:
     ipam:
       config:
         - subnet: 172.100.0.0/16
+
+volumes:
+  build-extension:

main.py

@@ -4,11 +4,13 @@ import time
 import socket
 import logging
 import os
+import argparse
 from yaspin import yaspin
 from functools import partial
 from http.server import HTTPServer
 from utils.PostHandler import Handler
 from utils.ColoredFormatter import ColoredFormatter
+from utils.Convenience import *
 from bs4 import BeautifulSoup as bs
 from selenium import webdriver
 from selenium.webdriver.chrome.options import Options
@@ -18,16 +20,29 @@ from selenium.webdriver.support.wait import WebDriverWait
 from selenium.webdriver.support import expected_conditions as ec
 
 logger = logging.getLogger(__name__)
+args = None
 
 def setupLogger():
     logging_format = "[%(asctime)s] (%(levelname)s) %(module)s - %(funcName)s: %(message)s"
-    logging.basicConfig(level=logging.INFO, format=logging_format)
+    logging.basicConfig(level=firstValid(args.log_level, os.getenv('LOG_LEVEL'), default='INFO'), format=logging_format) # type: ignore
     (logger := logging.getLogger(__name__)).setLevel(logging.INFO)
     logger.propagate = False
     (logger_handler := logging.StreamHandler()).setFormatter(
         ColoredFormatter(fmt=logging_format)
     )
     logger.addHandler(logger_handler)
+    
+
+def setupArgParser():
+    parser = argparse.ArgumentParser(description='Collector for PeerTube stats.')
+    parser.add_argument('-u', '--url', type=str, help='URL of the video to collect stats for.')
+    parser.add_argument('--socket-url', type=str, help='URL of the socket to send the stats to. Default: localhost')
+    parser.add_argument('--socket-port', type=int, help='Port of the socket to send the stats to. Default: 8094')
+    parser.add_argument('--hub-url', type=str, help='URL of the Selenium hub to connect to. If not provided, local Chrome driver will be used.')
+    parser.add_argument('--webrtc-internals-path', type=str, help='Path to the WebRTC internals extension.')
+    parser.add_argument('--log-level', type=str, help='Log level to use. Default: INFO')
+    
+    return parser
 
 def interrupt_handler(signum, driver: webdriver.Remote):
     logger.info(f'Handling signal {signum} ({signal.Signals(signum).name}).')
@@ -36,37 +51,40 @@ def interrupt_handler(signum, driver: webdriver.Remote):
     raise SystemExit
 
 @yaspin()
-def setupChromeDriver():
+def setupChromeDriver(command_executor: str | None, webrtc_internals_path: str) -> webdriver.Remote | webdriver.Chrome:
     logger.log(logging.INFO, 'Setting up Chrome driver.')
     chrome_options = Options()
     #chrome_options.add_argument("--headless")
     chrome_options.add_argument("--no-sandbox")
     chrome_options.add_argument("--mute-audio")
     chrome_options.add_argument("--window-size=1280,720")
-    #chrome_options.add_argument("--disable-dev-shm-usage")
     chrome_options.add_argument("--no-default-browser-check")
     chrome_options.add_argument("--disable-features=WebRtcHideLocalIpsWithMdns")
-    #chrome_options.add_argument(f"--load-extension={os.path.abspath(os.path.join(os.path.dirname(__file__), 'webrtc-internals-exporter'))}")
-    chrome_options.add_argument("--load-extension=/tmp/webrtc-internals-exporter")
+    chrome_options.add_argument(f"--load-extension={webrtc_internals_path}")
     chrome_options.add_experimental_option('prefs', {'intl.accept_languages': 'en,en_US'})
-        
-    #driver = webdriver.Chrome(options=chrome_options)
-    driver = webdriver.Remote(command_executor='http://host.docker.internal:4444', options=chrome_options)
+
+    if command_executor is not None:
+        driver = webdriver.Remote(command_executor=command_executor, options=chrome_options)
+        logger.warning(f'Using Selenium hub at {command_executor}.')
+    else:
+        driver = webdriver.Chrome(options=chrome_options)
+        logger.warning('No Selenium hub URL provided, using local Chrome driver.')
+    
     logger.log(logging.INFO, 'Chrome driver setup complete.')
     
     return driver
 
-def saveStats(stats: list):
+def saveStats(stats: list, socket_url: str, socket_port: int):
     try:
         sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
         logger.log(logging.DEBUG, f'Saving stats: {json.dumps(stats, indent=4)}')
-        sock.sendto(json.dumps(stats).encode(), ('telegraf', 8094))
+        sock.sendto(json.dumps(stats).encode(), (socket_url, socket_port))
         sock.close()
         logger.log(logging.DEBUG, 'Sent stats to socket.')
     except socket.error as e:
         logger.error(f'Got socket error: {e}')
 
-def downloadStats(driver: webdriver.Chrome, peersDict: dict):
+def downloadStats(driver: webdriver.Remote | webdriver.Chrome, peersDict: dict, socket_url: str, socket_port: int):
     html = driver.find_element(By.CLASS_NAME ,'vjs-stats-list').get_attribute('innerHTML')
     if html is not None:
         htmlBS = bs(html, 'html.parser')
@@ -157,7 +175,7 @@ def downloadStats(driver: webdriver.Chrome, peersDict: dict):
         'session': driver.session_id
     }
 
-    saveStats([stats])
+    saveStats([stats], socket_url, socket_port)
 
 def convert_to_bytes(down, downUnit):
     return float(down) * (1024 ** {'B': 0, 'KB': 1, 'MB': 2, 'GB': 3}[downUnit])
@@ -188,20 +206,31 @@ def setupStats(driver: webdriver.Remote, url: str):
     
     return driver
 
-if __name__ == '__main__':    
+if __name__ == '__main__':
+    args = setupArgParser().parse_args()
+    
     setupLogger()
     
-    driver = setupChromeDriver()
+    command_executor = firstValid(args.hub_url, os.getenv('HUB_URL'), default=None)
+    webrtc_internals_path = firstValid(
+        args.webrtc_internals_path,
+        os.getenv('WEBRTC_INTERNALS_PATH'), 
+        default=os.path.abspath(os.path.join(os.path.dirname(__file__), 'webrtc-internals-exporter'))
+        )
+    
+    driver = setupChromeDriver(command_executor, webrtc_internals_path)
 
     signal.signal(signal.SIGINT, lambda signum, frame: interrupt_handler(signum, driver))
     
-    url = os.getenv('VIDEO_URL')
+    url = firstValid(args.url, os.getenv('VIDEO_URL'), default=None)
     if url is None:
-        logger.error('VIDEO_URL environment variable is not set.')
+        logger.error('VIDEO_URL environment variable or --url argument is required.')
         raise SystemExit(1)
     
     setupStats(driver, url)
     
+    socket_url = firstValid(args.socket_url, os.getenv('SOCKET_URL'), default='localhost')
+    socket_port = firstValid(args.socket_port, os.getenv('SOCKET_PORT'), default=8094)
     logger.info('Starting server collector.')
-    httpd = HTTPServer(('', 9092), partial(Handler, downloadStats, driver, logger))
+    httpd = HTTPServer(('', 9092), partial(Handler, downloadStats, driver, logger, socket_url, socket_port))
     httpd.serve_forever()

monolith-entrypoint.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+if [ -z "$TELEGRAF_HOSTNAME" ]; then
+    echo "Error: TELEGRAF_HOSTNAME is not set"
+    exit 1
+fi
+
+if [ -z "$TELEGRAF_MONGODB_DSN" ]; then
+    echo "Error: TELEGRAF_MONGODB_DSN is not set"
+    exit 1
+fi
+
+if [ -z "$TELEGRAF_MONGODB_DATABASE" ]; then
+    echo "Error: TELEGRAF_MONGODB_DATABASE is not set"
+    exit 1
+fi
+
+if [ -z "$VIDEO_URL" ]; then
+    echo "Error: VIDEO_URL is not set"
+    exit 1
+fi
+
+# Set the environment variables
+export DSN=$TELEGRAF_MONGODB_DSN
+export DATABASE=$TELEGRAF_MONGODB_DATABASE
+export HOSTNAME=$TELEGRAF_HOSTNAME
+
+# Start the Selenium hub
+/opt/bin/entry_point.sh > /dev/null 2>&1 &
+
+# Wait for Selenium hub to be ready
+printf 'Waiting for Selenium standalone to be ready'
+timeout=30
+while ! curl -sSL "http://localhost:4444/wd/hub/status" 2>/dev/null | jq -e '.value.ready' | grep -q true; do
+    printf '.'
+    sleep 1
+    ((timeout--))
+    if [ $timeout -le 0 ]; then
+        echo "Error: Selenium standalone did not become ready in time. Exiting..."
+        exit 1
+    fi
+done
+
+printf '\n'
+
+# Start the Telegraf agent and the main script
+telegraf --config ./telegraf.conf &
+./venv/bin/python main.py

utils/Convenience.py

@@ -0,0 +1,5 @@
+def firstValid(*args, default):
+    for arg in args:
+        if arg is not None:
+            return arg
+    return default

utils/PostHandler.py

@@ -3,10 +3,12 @@ import logging
 from http.server import BaseHTTPRequestHandler
 
 class Handler(BaseHTTPRequestHandler):
-    def __init__(self, custom_func, driver, logger, *args, **kwargs):
+    def __init__(self, custom_func, driver, logger, socket_url, socket_port, *args, **kwargs):
         self._custom_func = custom_func
         self.logger = logger
         self.driver = driver
+        self._socket_url = socket_url
+        self._socket_port = socket_port
         super().__init__(*args, **kwargs)
         
     def do_POST(self):
@@ -14,7 +16,7 @@ class Handler(BaseHTTPRequestHandler):
             content_length = int(self.headers['Content-Length'])
             post_data = self.rfile.read(content_length)
             self.logger.log(logging.DEBUG, f"POST request,\nPath: {self.path}\nHeaders:\n{self.headers}\n\nBody:\n{post_data.decode('utf-8')}")
-            self._custom_func(self.driver, json.loads(post_data.decode('utf-8')))
+            self._custom_func(self.driver, json.loads(post_data.decode('utf-8')), self._socket_url, self._socket_port)
             self.send_response(200)
             self.end_headers()
             self.wfile.write(b'POST request received')

webrtc-internals-exporter/background.js

@@ -6,17 +6,17 @@ function log(...args) {
 
 log("loaded");
 
-import "/assets/pako.min.js";
+import "./assets/pako.min.js";
 
 const DEFAULT_OPTIONS = {
-  url: "http://localhost:9092",
+  url: process.env.WEBRTC_INTERNALS_EXPORTER_URL + ":9092",
   username: "",
   password: "",
   updateInterval: 2,
   gzip: false,
   job: "webrtc-internals-exporter",
-  enabledOrigins: { },
-  enabledStats: ["data-channel", "local-candidate", "remote-candidate"]
+  enabledOrigins: {},
+  enabledStats: ["data-channel", "local-candidate", "remote-candidate", "candidate-pair"]
 };
 
 const options = {};

webrtc-internals-exporter/manifest.json

@@ -40,7 +40,7 @@
     }
   ],
   "background": {
-    "service_worker": "background.js",
+    "service_worker": "background.bundle.js",
     "type": "module"
   },
   "web_accessible_resources": [

webrtc-internals-exporter/webpack/babel.config.js

@@ -0,0 +1,3 @@
+module.exports = {
+  shouldPrintComment: () => false
+};

webrtc-internals-exporter/webpack/build.js

@@ -0,0 +1,26 @@
+const { execSync } = require('child_process');
+const args = process.argv.slice(2);
+
+let url = '';
+
+args.forEach((arg, index) => {
+    if (arg === '-u' || arg === '--url') {
+        url = args[index + 1];
+    } else if (arg === '-h' || arg === '--help') {
+        console.log('Usage: npm run build -- [-u|--url <url>]');
+        console.log('Options:');
+        console.log('  -u, --url <url>  URL to use for the extension collector server');
+        console.log('  -h, --help       Display this help message');
+        process.exit(0);
+    } else if (arg.startsWith('-')) {
+        console.error(`Unrecognized argument: ${arg}`);
+        process.exit(1);
+    }
+});
+
+if (url) {
+  console.log(`Building with URL: ${url}`);
+  execSync(`webpack --env URL=${url}`, { stdio: 'inherit' });
+} else {
+  execSync('webpack', { stdio: 'inherit' });
+}

webrtc-internals-exporter/webpack/package.json

@@ -0,0 +1,32 @@
+{
+  "name": "webrtc-internals-exporter",
+  "version": "1.0.0",
+  "private": true,
+  "scripts": {
+    "build": "node build.js"
+  },
+  "keywords": [],
+  "author": "Mirko Milovanovic",
+  "license": "MIT",
+  "devDependencies": {
+    "babel-loader": "^8.2.2",
+    "clean-webpack-plugin": "^3.0.0",
+    "copy-webpack-plugin": "^12.0.2",
+    "file-loader": "^6.2.0",
+    "html-webpack-plugin": "^5.3.1",
+    "mini-css-extract-plugin": "^1.6.0",
+    "postcss": "^8.2.14",
+    "postcss-loader": "^5.2.0",
+    "postcss-preset-env": "^10.1.4",
+    "sass": "^1.32.12",
+    "sass-loader": "^11.0.1",
+    "serve": "^14.2.4",
+    "style-loader": "^2.0.0",
+    "terser-webpack-plugin": "^5.1.1",
+    "ts-loader": "^9.1.2",
+    "typescript": "^4.2.4",
+    "webpack": "^5.38.1",
+    "webpack-cli": "^4.7.2",
+    "webpack-dev-server": "^5.2.0"
+  }
+}

webrtc-internals-exporter/webpack/webpack.config.js

@@ -0,0 +1,32 @@
+const path = require('path');
+const { EnvironmentPlugin } = require('webpack');
+
+module.exports = (env) => {
+  const url = env.URL || 'http://localhost';
+
+  return {
+    entry: '../background.js',
+    target: 'web',
+    mode: 'production',
+    module: {
+      rules: [
+        {
+          test: /\.js?$/,
+          use: 'babel-loader',
+          exclude: /node_modules/,
+        },
+      ],
+    },
+    resolve: { extensions: ['.tsx', '.ts', '.js'] },
+    output: {
+      filename: 'background.bundle.js',
+      path: path.resolve(__dirname, '../'),
+      publicPath: '',
+    },
+    plugins: [
+      new EnvironmentPlugin({
+        WEBRTC_INTERNALS_EXPORTER_URL: url,
+      }),
+    ],
+  };
+};