kobim/Tesi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Add Docker support and update requirements for data collection and visualization

Browse Source
This commit is contained in:
Mirko Milovanovic
2025-02-05 01:44:39 +01:00
parent df0cbaffbd
commit ad97310d7e
11 changed files with 92 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
peertube/statnerd/.dockerignore Normal file
View File

@@ -0,0 +1,21 @@
__pycache__
*.pyc
*.pyo
*.pyd
.Python
env
pip-log.txt
pip-delete-this-directory.txt
.tox
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.log
.git
.mypy_cache
.pytest_cache
.hypothesis
.pytest_caches

63
peertube/statnerd/CRUD/mongodb-convert-to.ts.js
View File

@@ -1,63 +0,0 @@
/* global use, db */
// MongoDB Playground
// To disable this template go to Settings | MongoDB | Use Default Template For Playground.
// Make sure you are connected to enable completions and to be able to run a playground.
// Use Ctrl+Space inside a snippet or a string literal to trigger completions.
// The result of the last command run in a playground is shown on the results panel.
// By default the first 20 documents will be returned with a cursor.
// Use 'console.log()' to print to the debug output.
// For more documentation on playgrounds please refer to
// https://www.mongodb.com/docs/mongodb-vscode/playgrounds/
use("statistics");
db.peertube.aggregate([
{
$set: {
player: {
$function: {
lang: "js",
args: ["$player"],
body: function (str) {
try {
return JSON.parse(str);
} catch (e) {
if (typeof str === "object") {
return str;
} else {
return null;
}
}
},
},
},
peers: {
$function: {
lang: "js",
args: ["$peers"],
body: function (str) {
try {
return JSON.parse(str);
} catch (e) {
if (typeof str === "object") {
return str;
} else {
return null;
}
}
},
},
},
},
},
{
$out: {
db: "statistics",
coll: "peertube_ts",
timeseries: {
timeField: "timestamp",
metaField: "tags",
granularity: "seconds",
},
},
},
]);

141
peertube/statnerd/CRUD/sessions-metrics.js
View File

@@ -1,141 +0,0 @@
/* global use, db */
// MongoDB Playground
// To disable this template go to Settings | MongoDB | Use Default Template For Playground.
// Make sure you are connected to enable completions and to be able to run a playground.
// Use Ctrl+Space inside a snippet or a string literal to trigger completions.
// The result of the last command run in a playground is shown on the results panel.
// By default the first 20 documents will be returned with a cursor.
// Use 'console.log()' to print to the debug output.
// For more documentation on playgrounds please refer to
// https://www.mongodb.com/docs/mongodb-vscode/playgrounds/
use("statistics");
let formattedDate = (date) => ({
unix: { $toLong: date },
iso: { $toString: date },
});
db.getCollection("peertube_ts").aggregate([
{
$sort: { timestamp: 1 }
},
{
$group: {
_id: "$tags.session",
host: { $first: "$tags.host" },
firstTimestamp: { $first: "$timestamp" },
lastTimestamp: { $last: "$timestamp" },
firstTimestampWithPeers: {
$min: {
$cond: {
if: { $gt: [{ $size: { $ifNull: ["$peers", []] } }, 0] },
then: "$timestamp",
else: null,
}
}
},
maxNumberOfPeers: {
$max: {
$size: { $ifNull: ["$peers", []] },
}
},
minNumberOfPeers: {
$min: {
$size: { $ifNull: ["$peers", []] },
}
}
}
},
// Lookup other sessions that may have peers
{
$lookup: {
from: "peertube_ts",
let: { currentSession: "$_id", ftp: "$firstTimestampWithPeers", fst: "$firstTimestamp" },
pipeline: [
{
$match: {
$expr: {
$and: [
{ $ne: ["$tags.session", "$$currentSession"] },
{ $lt: ["$timestamp", "$$ftp"] },
{ $gte: ["$timestamp", "$$fst"] },
{ $gt: [{ $size: { $ifNull: ["$peers", []] } }, 0] }
]
}
}
}
],
as: "concurrentSessions"
}
},
// Mark whether peers existed in other sessions before starting
{
$addFields: {
concurrentSessions: {
$gt: [{ $size: "$concurrentSessions" }, 0]
}
}
},
{
$group: {
_id: "$host",
sessions: {
$push: {
id: "$_id",
startTime: formattedDate("$firstTimestamp"),
endTime: formattedDate("$lastTimestamp"),
duration: {
$divide: [
{
$subtract: ["$lastTimestamp", "$firstTimestamp"],
},
1000,
]
},
firstPeerConnection: {
$cond: {
if: { $eq: ["$firstTimestampWithPeers", null] },
then: null,
else: {
time: {
date: formattedDate("$firstTimestampWithPeers"),
elapsedFromStart: {
$divide: [
{
$subtract: ["$firstTimestampWithPeers", "$firstTimestamp"],
},
1000,
]
}
},
concurrentSessions: "$concurrentSessions",
}
}
},
maxPeers: { $max: "$maxNumberOfPeers" },
minPeers: { $min: "$minNumberOfPeers" }
}
}
}
},
{
$set: {
sessions: {
$sortArray: {
input: "$sessions",
sortBy: { id: 1 }
}
}
}
},
{
$project: {
_id: 0,
host: "$_id",
sessions: "$sessions"
}
},
{
$sort: { host: 1 }
}
]);

13
peertube/statnerd/Dockerfile Normal file
View File

@@ -0,0 +1,13 @@
FROM python:3.13.1-slim-bookworm
# Install dependencies
COPY requirements.txt /app/requirements.txt
RUN pip install --no-cache-dir -r /app/requirements.txt
# Copy the application
COPY main.py /app
COPY utils/ /app/utils
WORKDIR /app
# Run the application
CMD ["python", "main.py"]

17
peertube/statnerd/README.md
View File

@@ -1,17 +0,0 @@
## Statistiche per nerd da Peer Tube
spero funzioni
per installarlo baste creare un nuovo ambiente e installare le dipendenze con:
```
python -m venv env
source .\env\bin\activate
pip install -r requirements.txt
```
per eseguire il tutto basta lanciare:
```
jupyter-lab
```

41
peertube/statnerd/docker-compose.yml Normal file
View File

@@ -0,0 +1,41 @@
services:
selenium:
container_name: selenium-standalone-chromium
image: selenium/standalone-chromium:129.0
ports:
- "7900:7900"
volumes:
- ./webrtc-internals-exporter:/tmp/webrtc-internals-exporter:ro
shm_size: "2g"
attach: false
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:4444/wd/hub/status"]
interval: 5s
timeout: 10s
retries: 5
networks:
- backend
telegraf:
container_name: telegraf
image: telegraf:1.33.1
volumes:
- ./telegraf.conf:/etc/telegraf/telegraf.conf:ro
networks:
- backend
collector:
container_name: collector
build:
context: .
dockerfile: Dockerfile
depends_on:
selenium:
condition: service_healthy
telegraf:
condition: service_started
networks:
- backend
networks:
backend:

13
peertube/statnerd/main.py
View File

@@ -44,11 +44,12 @@ def setupChromeDriver():
chrome_options.add_argument("--disable-dev-shm-usage")
chrome_options.add_argument("--no-default-browser-check")
chrome_options.add_argument("--disable-features=WebRtcHideLocalIpsWithMdns")
chrome_options.add_argument(f"--load-extension={os.path.abspath(os.path.join(os.path.dirname(__file__), 'webrtc-internals-exporter'))}")
#chrome_options.add_argument(f"--load-extension={os.path.abspath(os.path.join(os.path.dirname(__file__), 'webrtc-internals-exporter'))}")
chrome_options.add_argument("--load-extension=/tmp/webrtc-internals-exporter")
chrome_options.add_experimental_option('prefs', {'intl.accept_languages': 'en,en_US'})
driver = webdriver.Chrome(options=chrome_options)
#driver = webdriver.Remote(command_executor='http://localhost:4444', options=chrome_options)
#driver = webdriver.Chrome(options=chrome_options)
driver = webdriver.Remote(command_executor='http://selenium-standalone-chromium:4444', options=chrome_options)
logger.log(logging.INFO, 'Chrome driver setup complete.')
return driver
@@ -57,7 +58,7 @@ def saveStats(stats: list):
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
logger.log(logging.DEBUG, f'Saving stats: {json.dumps(stats, indent=4)}')
sock.sendto(json.dumps(stats).encode(), ('localhost', 8094))
sock.sendto(json.dumps(stats).encode(), ('telegraf', 8094))
sock.close()
logger.log(logging.DEBUG, 'Sent stats to socket.')
except socket.error as e:
@@ -185,6 +186,6 @@ if __name__ == '__main__':
setupStats(driver, "https://tube.kobim.cloud/w/iN2T8PmbSb4HJTDA2rV3sg")
logger.log(logging.INFO, 'Starting server collector.')
httpd = HTTPServer(('localhost', 9092), partial(Handler, downloadStats, driver, logger))
httpd = HTTPServer(('collector', 9092), partial(Handler, downloadStats, driver, logger))
logger.info('Server collector started.')
httpd.serve_forever()

3773
peertube/statnerd/plot.ipynb
View File

File diff suppressed because one or more lines are too long

10
peertube/statnerd/requirements.txt
View File

@@ -1,10 +1,2 @@
selenium
beautifulsoup4
pymongo
pandas
matplotlib
seaborn
numpy
scipy
plotly
nbformat
beautifulsoup4

2
peertube/statnerd/webrtc-internals-exporter/background.js
View File

@@ -9,7 +9,7 @@ log("loaded");
import "/assets/pako.min.js";
const DEFAULT_OPTIONS = {
url: "http://localhost:9092",
url: "http://collector:9092",
username: "",
password: "",
updateInterval: 2,