From 7e02c0ad782fc7cde06eb1712ce50f4a02e02ebc Mon Sep 17 00:00:00 2001 From: Luc VOISE Date: Fri, 16 Oct 2020 15:30:08 +0200 Subject: [PATCH 01/27] Add specific server id --- Dockerfile | 2 +- main.py | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7b508ef..888fa80 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM python:3.8-slim-buster -LABEL maintainer="Josh Smith" \ +LABEL maintainer="Luc VOISE" \ description="Original by Aiden Gilmartin. Speedtest to InfluxDB data bridge" # Install dependencies diff --git a/main.py b/main.py index 9c3b4b2..f2ac46c 100755 --- a/main.py +++ b/main.py @@ -17,6 +17,8 @@ DB_TAGS = os.environ.get('INFLUX_DB_TAGS') TEST_INTERVAL = int(os.environ.get('SPEEDTEST_INTERVAL')) * 60 # Time before retrying a failed Speedtest (in minutes, converts to seconds). TEST_FAIL_INTERVAL = int(os.environ.get('SPEEDTEST_FAIL_INTERVAL')) * 60 +# Specific server ID +SERVER_ID = os.environ.get('SPEEDTEST_SERVER_ID') influxdb_client = InfluxDBClient( DB_ADDRESS, DB_PORT, DB_USER, DB_PASSWORD, None) @@ -125,9 +127,15 @@ def main(): init_db() # Setup the database if it does not already exist. while (1): # Run a Speedtest and send the results to influxDB indefinitely. - speedtest = subprocess.run( + server_id = SERVER_ID + if not server_id: + speedtest = subprocess.run( ["speedtest", "--accept-license", "--accept-gdpr", "-f", "json"], capture_output=True) - + print("Automatic server choice") + else : + speedtest = subprocess.run( + ["speedtest", "--accept-license", "--accept-gdpr", "-f", "json", "--server-id=" + SERVER_ID], capture_output=True) + print("Manual server choice : ID = " + SERVER_ID) if speedtest.returncode == 0: # Speedtest was successful. data = format_for_influx(speedtest.stdout) print("Speedtest Successful:") From 2e145441851c31d2bd7f800d733c3d26a8c4f6cf Mon Sep 17 00:00:00 2001 From: Sky007FR Date: Fri, 16 Oct 2020 15:57:48 +0200 Subject: [PATCH 02/27] Update README.md --- README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index cc8984f..0656748 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ The variables available are: - INFLUX_DB_TAGS = *comma seperated list of tags. See below for options* - SPEEDTEST_INTERVAL = 60 - SPEEDTEST_FAIL_INTERVAL = 5 +- SPEEDTEST_SERVER_ID = *id from https://c.speedtest.net/speedtest-servers-static.php* ### Variable Notes - Intervals are in minutes. *Script will convert it to seconds.* @@ -58,7 +59,7 @@ Be aware that this script will automatically accept the license and GDPR stateme 1. Build the container. - `docker build -t breadlysm/speedtest-to-influxdb ./` + `docker build -t sky007fr/speedtest-to-influxdb ./` 2. Run the container. ``` @@ -70,13 +71,14 @@ Be aware that this script will automatically accept the license and GDPR stateme -e 'INFLUX_DB_DATABASE'='speedtest' \ -e 'SPEEDTEST_INTERVAL'='1800' \ -e 'SPEEDTEST_FAIL_INTERVAL'='60' \ - breadlysm/speedtest-to-influxdb + -e 'SPEEDTEST_SERVER_ID'='12746' \ + sky007fr/speedtest-to-influxdb ``` ### No Container 1. Clone the repo - `git clone https://github.com/breadlysm/speedtest-to-influxdb.git` + `git clone https://github.com/sky007fr/speedtest-to-influxdb.git` 2. Configure the .env file in the repo or set the environment variables on your device. @@ -94,4 +96,4 @@ Be aware that this script will automatically accept the license and GDPR stateme -This script looks to have been originally written by https://github.com/aidengilmartin/speedtest-to-influxdb/blob/master/main.py and I forked it from https://github.com/martinfrancois/speedtest-to-influxdb. They did the hard work, I've continued to modify it though to fit my needs. +This script looks to have been originally written by https://github.com/aidengilmartin/speedtest-to-influxdb/blob/master/main.py and I forked it from https://github.com/breadlysm/speedtest-to-influxdb. They did the hard work, I've continued to modify it though to fit my needs. From 34c71dce652238123462f26be6cb2d69eb3e20d0 Mon Sep 17 00:00:00 2001 From: Luc VOISE Date: Fri, 16 Oct 2020 17:20:09 +0200 Subject: [PATCH 03/27] Add print for log debug --- main.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index f2ac46c..5f3ddbf 100755 --- a/main.py +++ b/main.py @@ -138,17 +138,18 @@ def main(): print("Manual server choice : ID = " + SERVER_ID) if speedtest.returncode == 0: # Speedtest was successful. data = format_for_influx(speedtest.stdout) - print("Speedtest Successful:") + print("Speedtest Successful :") + print(speedtest.stdout) if influxdb_client.write_points(data) == True: print("Data written to DB successfully") time.sleep(TEST_INTERVAL) else: # Speedtest failed. - print("Speedtest Failed:") + print("Speedtest Failed :") print(speedtest.stderr) print(speedtest.stdout) time.sleep(TEST_FAIL_INTERVAL) if __name__ == '__main__': - print('Speedtest CLI Data Logger to InfluxDB') + print('Speedtest CLI data logger to InfluxDB started...') main() From 99bc5b3bd46fb8f147efdfffa9a984acfd19ac3a Mon Sep 17 00:00:00 2001 From: Leigh Phillips Date: Sat, 16 Jan 2021 13:06:25 -0800 Subject: [PATCH 04/27] Create entrypoint.sh --- entrypoint.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 entrypoint.sh diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100644 index 0000000..6bd11aa --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +printenv >> /etc/environment +ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +# Install speedtest-cli +apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 379CE192D401AB61 +echo "deb https://ookla.bintray.com/debian buster main" | tee /etc/apt/sources.list.d/speedtest.list +apt-get update && apt-get -q -y install speedtest +#apt-get -q -y autoremove && apt-get -q -y clean +#rm -rf /var/lib/apt/lists/* + +exec /usr/local/bin/python3 $@ From cb3eb1be7f84040a5d9a8530b6455e3ba8e93925 Mon Sep 17 00:00:00 2001 From: Leigh Phillips Date: Sat, 16 Jan 2021 13:06:50 -0800 Subject: [PATCH 05/27] Delete requirements.txt --- requirements.txt | 1 - 1 file changed, 1 deletion(-) delete mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 1d4ac6e..0000000 --- a/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -influxdb \ No newline at end of file From 61fb17ea67efe2728b8e2f2df19c1e0ba658869d Mon Sep 17 00:00:00 2001 From: Leigh Phillips Date: Sat, 16 Jan 2021 13:15:21 -0800 Subject: [PATCH 06/27] Update Dockerfile Make hub eligible - Remove any proprietary binaries (Oookla) & install those at container initialization instead. --- Dockerfile | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/Dockerfile b/Dockerfile index 888fa80..821207e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,28 +1,24 @@ FROM python:3.8-slim-buster - -LABEL maintainer="Luc VOISE" \ +LABEL maintainer="Team QLUSTOR " \ description="Original by Aiden Gilmartin. Speedtest to InfluxDB data bridge" -# Install dependencies ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update -RUN apt-get -q -y install --no-install-recommends apt-utils gnupg1 apt-transport-https dirmngr - -# Install speedtest-cli -RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 379CE192D401AB61 -RUN echo "deb https://ookla.bintray.com/debian buster main" | tee /etc/apt/sources.list.d/speedtest.list -RUN apt-get update && apt-get -q -y install speedtest +RUN true &&\ +\ +# Install dependencies +apt-get update && \ +apt-get -q -y install --no-install-recommends apt-utils gnupg1 apt-transport-https dirmngr && \ +\ # Install Python packages -COPY requirements.txt / -RUN pip install -r /requirements.txt - +pip3 install influxdb && \ +\ # Clean up -RUN apt-get -q -y autoremove -RUN apt-get -q -y clean -RUN rm -rf /var/lib/apt/lists/* +apt-get -q -y autoremove && apt-get -q -y clean && \ +rm -rf /var/lib/apt/lists/* # Final setup & execution -COPY . /app +ADD . /app WORKDIR /app -CMD ["python3", "-u", "main.py"] \ No newline at end of file +ENTRYPOINT ["/bin/sh", "/app/entrypoint.sh"] +CMD ["main.py"] From 74ae03ba9fe810d88b266f24b8cc4f1eedfd6cde Mon Sep 17 00:00:00 2001 From: Leigh Phillips Date: Sat, 16 Jan 2021 13:41:46 -0800 Subject: [PATCH 07/27] Update main.py Add defaults for ENVs and NAMESPACE --- main.py | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/main.py b/main.py index 8c4395b..8b77c8b 100755 --- a/main.py +++ b/main.py @@ -4,21 +4,29 @@ import subprocess import os from influxdb import InfluxDBClient +func getEnv(key, fallback string) string { + if value, ok := os.LookupEnv(key); ok { + return value + } + return fallback +} + # InfluxDB Settings -DB_ADDRESS = os.environ.get('INFLUX_DB_ADDRESS') -DB_PORT = int(os.environ.get('INFLUX_DB_PORT')) -DB_USER = os.environ.get('INFLUX_DB_USER') -DB_PASSWORD = os.environ.get('INFLUX_DB_PASSWORD') -DB_DATABASE = os.environ.get('INFLUX_DB_DATABASE') -DB_TAGS = os.environ.get('INFLUX_DB_TAGS') +NAMESPACE = getEnv('NAMESPACE', '') +DB_ADDRESS = getEnv('INFLUX_DB_ADDRESS', 'influxdb') +DB_PORT = int(getEnv('INFLUX_DB_PORT', '8086')) +DB_USER = getEnv('INFLUX_DB_USER', '') +DB_PASSWORD = getEnv('INFLUX_DB_PASSWORD', '') +DB_DATABASE = getEnv('INFLUX_DB_DATABASE', 'speedtests') +DB_TAGS = getEnv('INFLUX_DB_TAGS', '') # Speedtest Settings # Time between tests (in minutes, converts to seconds). -TEST_INTERVAL = int(os.environ.get('SPEEDTEST_INTERVAL')) * 60 +TEST_INTERVAL = int(getEnv('SPEEDTEST_INTERVAL', '5')) * 60 # Time before retrying a failed Speedtest (in minutes, converts to seconds). -TEST_FAIL_INTERVAL = int(os.environ.get('SPEEDTEST_FAIL_INTERVAL')) * 60 +TEST_FAIL_INTERVAL = int(getEnv('SPEEDTEST_FAIL_INTERVAL', '5')) * 60 # Specific server ID -SERVER_ID = os.environ.get('SPEEDTEST_SERVER_ID') +SERVER_ID = getEnv('SPEEDTEST_SERVER_ID', '') influxdb_client = InfluxDBClient( DB_ADDRESS, DB_PORT, DB_USER, DB_PASSWORD, None) @@ -48,6 +56,7 @@ def tag_selection(data): return None # tag_switch takes in _data and attaches CLIoutput to more readable ids tag_switch = { + 'namespace': NAMESPACE, 'isp': data['isp'], 'interface': data['interface']['name'], 'internal_ip': data['interface']['internalIp'], @@ -66,6 +75,7 @@ def tag_selection(data): } options = {} + tags = 'namespace, ' + tags tags = tags.split(',') for tag in tags: # split the tag string, strip and add selected tags to {options} with corresponding tag_switch data @@ -82,6 +92,7 @@ def format_for_influx(data): 'measurement': 'ping', 'time': data['timestamp'], 'fields': { + 'namespace': NAMESPACE, 'jitter': data['ping']['jitter'], 'latency': data['ping']['latency'] } @@ -90,6 +101,7 @@ def format_for_influx(data): 'measurement': 'download', 'time': data['timestamp'], 'fields': { + 'namespace': NAMESPACE, # Byte to Megabit 'bandwidth': data['download']['bandwidth'] / 125000, 'bytes': data['download']['bytes'], @@ -100,6 +112,7 @@ def format_for_influx(data): 'measurement': 'upload', 'time': data['timestamp'], 'fields': { + 'namespace': NAMESPACE, # Byte to Megabit 'bandwidth': data['upload']['bandwidth'] / 125000, 'bytes': data['upload']['bytes'], @@ -110,6 +123,7 @@ def format_for_influx(data): 'measurement': 'packetLoss', 'time': data['timestamp'], 'fields': { + 'namespace': NAMESPACE, 'packetLoss': pkt_loss(data) } } @@ -154,4 +168,4 @@ def main(): if __name__ == '__main__': print('Speedtest CLI data logger to InfluxDB started...') - main() \ No newline at end of file + main() From c54e30c2008fda87819365d6f0496f9a8c8eb462 Mon Sep 17 00:00:00 2001 From: Leigh Phillips Date: Sat, 16 Jan 2021 13:56:40 -0800 Subject: [PATCH 08/27] Update main.py Try this again. --- main.py | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/main.py b/main.py index 8b77c8b..34a0d94 100755 --- a/main.py +++ b/main.py @@ -4,29 +4,22 @@ import subprocess import os from influxdb import InfluxDBClient -func getEnv(key, fallback string) string { - if value, ok := os.LookupEnv(key); ok { - return value - } - return fallback -} - # InfluxDB Settings -NAMESPACE = getEnv('NAMESPACE', '') -DB_ADDRESS = getEnv('INFLUX_DB_ADDRESS', 'influxdb') -DB_PORT = int(getEnv('INFLUX_DB_PORT', '8086')) -DB_USER = getEnv('INFLUX_DB_USER', '') -DB_PASSWORD = getEnv('INFLUX_DB_PASSWORD', '') -DB_DATABASE = getEnv('INFLUX_DB_DATABASE', 'speedtests') -DB_TAGS = getEnv('INFLUX_DB_TAGS', '') +NAMESPACE = os.getenv('NAMESPACE', '') +DB_ADDRESS = os.getenv('INFLUX_DB_ADDRESS', 'influxdb') +DB_PORT = int(os.getenv('INFLUX_DB_PORT', '8086')) +DB_USER = os.getenv('INFLUX_DB_USER', '') +DB_PASSWORD = os.getenv('INFLUX_DB_PASSWORD', '') +DB_DATABASE = os.getenv('INFLUX_DB_DATABASE', 'speedtests') +DB_TAGS = os.getenv('INFLUX_DB_TAGS', '') # Speedtest Settings # Time between tests (in minutes, converts to seconds). -TEST_INTERVAL = int(getEnv('SPEEDTEST_INTERVAL', '5')) * 60 +TEST_INTERVAL = int(os.getenv('SPEEDTEST_INTERVAL', '5')) * 60 # Time before retrying a failed Speedtest (in minutes, converts to seconds). -TEST_FAIL_INTERVAL = int(getEnv('SPEEDTEST_FAIL_INTERVAL', '5')) * 60 +TEST_FAIL_INTERVAL = int(os.getenv('SPEEDTEST_FAIL_INTERVAL', '5')) * 60 # Specific server ID -SERVER_ID = getEnv('SPEEDTEST_SERVER_ID', '') +SERVER_ID = os.getenv('SPEEDTEST_SERVER_ID', '') influxdb_client = InfluxDBClient( DB_ADDRESS, DB_PORT, DB_USER, DB_PASSWORD, None) From 61d3405928570e1338e82add6b22cd3922326d85 Mon Sep 17 00:00:00 2001 From: Leigh Phillips Date: Sat, 16 Jan 2021 14:55:54 -0800 Subject: [PATCH 09/27] Update main.py Do not need namespace in tags. --- main.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/main.py b/main.py index 34a0d94..1e428dd 100755 --- a/main.py +++ b/main.py @@ -11,7 +11,7 @@ DB_PORT = int(os.getenv('INFLUX_DB_PORT', '8086')) DB_USER = os.getenv('INFLUX_DB_USER', '') DB_PASSWORD = os.getenv('INFLUX_DB_PASSWORD', '') DB_DATABASE = os.getenv('INFLUX_DB_DATABASE', 'speedtests') -DB_TAGS = os.getenv('INFLUX_DB_TAGS', '') +DB_TAGS = os.getenv('INFLUX_DB_TAGS', None) # Speedtest Settings # Time between tests (in minutes, converts to seconds). @@ -49,7 +49,6 @@ def tag_selection(data): return None # tag_switch takes in _data and attaches CLIoutput to more readable ids tag_switch = { - 'namespace': NAMESPACE, 'isp': data['isp'], 'interface': data['interface']['name'], 'internal_ip': data['interface']['internalIp'], @@ -68,7 +67,6 @@ def tag_selection(data): } options = {} - tags = 'namespace, ' + tags tags = tags.split(',') for tag in tags: # split the tag string, strip and add selected tags to {options} with corresponding tag_switch data From 981fdad9a83c1a606c0e9f0f319e48338f3c384c Mon Sep 17 00:00:00 2001 From: Leigh Phillips Date: Sat, 16 Jan 2021 15:29:59 -0800 Subject: [PATCH 10/27] Update main.py Add * for tags wildcard & default NAMESPACE to None. --- main.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/main.py b/main.py index 1e428dd..57673b3 100755 --- a/main.py +++ b/main.py @@ -5,7 +5,7 @@ import os from influxdb import InfluxDBClient # InfluxDB Settings -NAMESPACE = os.getenv('NAMESPACE', '') +NAMESPACE = os.getenv('NAMESPACE', 'None') DB_ADDRESS = os.getenv('INFLUX_DB_ADDRESS', 'influxdb') DB_PORT = int(os.getenv('INFLUX_DB_PORT', '8086')) DB_USER = os.getenv('INFLUX_DB_USER', '') @@ -66,12 +66,16 @@ def tag_selection(data): 'speedtest_url': data['result']['url'] } + if tags == '*': + return tag_switch + options = {} tags = tags.split(',') for tag in tags: # split the tag string, strip and add selected tags to {options} with corresponding tag_switch data tag = tag.strip() options[tag] = tag_switch[tag] + return options From b8ddba245b41189899001134d8068127e0fdb5a1 Mon Sep 17 00:00:00 2001 From: Leigh Phillips Date: Sat, 16 Jan 2021 17:17:35 -0800 Subject: [PATCH 11/27] Update README.md --- README.md | 55 ++++++++++++++++++------------------------------------- 1 file changed, 18 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index e5dce73..9204c89 100644 --- a/README.md +++ b/README.md @@ -15,15 +15,16 @@ There are some added features to allow some additional details that Ookla provid The InfluxDB connection settings are controlled by environment variables. The variables available are: -- INFLUX_DB_ADDRESS = 192.168.1.xxx -- INFLUX_DB_PORT = 8086 -- INFLUX_DB_USER = user -- INFLUX_DB_PASSWORD = pass -- INFLUX_DB_DATABASE = speedtest -- INFLUX_DB_TAGS = *comma seperated list of tags. See below for options* -- SPEEDTEST_INTERVAL = 60 -- SPEEDTEST_FAIL_INTERVAL = 5 -- SPEEDTEST_SERVER_ID = *id from https://c.speedtest.net/speedtest-servers-static.php* +- NAMESPACE = default - None +- INFLUX_DB_ADDRESS = default - influxdb +- INFLUX_DB_PORT = default - 8086 +- INFLUX_DB_USER = default - {blank} +- INFLUX_DB_PASSWORD = default - {blank} +- INFLUX_DB_DATABASE = default - speedtests +- INFLUX_DB_TAGS = default - None * See below for options, '*' widcard for all * +- SPEEDTEST_INTERVAL = default - 5 +- SPEEDTEST_FAIL_INTERVAL = deafult - 5 +- SPEEDTEST_SERVER_ID = default - {blank} * id from https://c.speedtest.net/speedtest-servers-static.php * ### Variable Notes - Intervals are in minutes. *Script will convert it to seconds.* @@ -60,41 +61,21 @@ Be aware that this script will automatically accept the license and GDPR stateme 1. Build the container. - `docker build -t sky007fr/speedtest-to-influxdb ./` + `docker build -t qlustor/speedtest_ookla-to-influxdb ./` 2. Run the container. ``` - docker run -d --name speedtest-influx \ - -e 'INFLUX_DB_ADDRESS'='_influxdb_host_' \ + docker run -d -t --name speedflux \ + -e 'NAMESPACE'='None' \ + -e 'INFLUX_DB_ADDRESS'='influxdb' \ -e 'INFLUX_DB_PORT'='8086' \ -e 'INFLUX_DB_USER'='_influx_user_' \ -e 'INFLUX_DB_PASSWORD'='_influx_pass_' \ - -e 'INFLUX_DB_DATABASE'='speedtest' \ - -e 'SPEEDTEST_INTERVAL'='1800' \ - -e 'SPEEDTEST_FAIL_INTERVAL'='60' \ + -e 'INFLUX_DB_DATABASE'='speedtests' \ + -e 'SPEEDTEST_INTERVAL'='5' \ + -e 'SPEEDTEST_FAIL_INTERVAL'='5' \ -e 'SPEEDTEST_SERVER_ID'='12746' \ - sky007fr/speedtest-to-influxdb + qlustor/speedtest_ookla-to-influxdb ``` -### No Container - -1. Clone the repo - - `git clone https://github.com/sky007fr/speedtest-to-influxdb.git` - -2. Configure the .env file in the repo or set the environment variables on your device. - -3. [Install the Speedtest CLI application by Ookla.](https://www.speedtest.net/apps/cli) - - NOTE: The `speedtest-cli` package in distro repositories is an unofficial client. It will need to be uninstalled before installing the Ookla Speedtest CLI application with the directions on their website. - -4. Install the InfluxDB client for library from Python. - - `pip install influxdb` - -5. Run the script. - - `python3 ./main.py` - - This script looks to have been originally written by https://github.com/aidengilmartin/speedtest-to-influxdb/blob/master/main.py and I forked it from https://github.com/breadlysm/speedtest-to-influxdb. They did the hard work, I've continued to modify it though to fit my needs. From adc0a4a1dd68e6ba1b3ea77e98cac90d9b073b99 Mon Sep 17 00:00:00 2001 From: Leigh Phillips Date: Sat, 16 Jan 2021 20:15:40 -0800 Subject: [PATCH 12/27] Update main.py Migrate namespace to tag. --- main.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/main.py b/main.py index 57673b3..99fd4d2 100755 --- a/main.py +++ b/main.py @@ -45,11 +45,12 @@ def pkt_loss(data): def tag_selection(data): tags = DB_TAGS - if tags is None: - return None + options = {} + # tag_switch takes in _data and attaches CLIoutput to more readable ids tag_switch = { - 'isp': data['isp'], + 'namespace': NAMESPACE, + 'isp': data['isp'], 'interface': data['interface']['name'], 'internal_ip': data['interface']['internalIp'], 'interface_mac': data['interface']['macAddr'], @@ -66,10 +67,13 @@ def tag_selection(data): 'speedtest_url': data['result']['url'] } - if tags == '*': + if tags is None: + tags = 'namespace' + elif '*' in tags: return tag_switch + else: + tags = 'namespace, ' + tags - options = {} tags = tags.split(',') for tag in tags: # split the tag string, strip and add selected tags to {options} with corresponding tag_switch data @@ -87,7 +91,6 @@ def format_for_influx(data): 'measurement': 'ping', 'time': data['timestamp'], 'fields': { - 'namespace': NAMESPACE, 'jitter': data['ping']['jitter'], 'latency': data['ping']['latency'] } @@ -96,7 +99,6 @@ def format_for_influx(data): 'measurement': 'download', 'time': data['timestamp'], 'fields': { - 'namespace': NAMESPACE, # Byte to Megabit 'bandwidth': data['download']['bandwidth'] / 125000, 'bytes': data['download']['bytes'], @@ -107,7 +109,6 @@ def format_for_influx(data): 'measurement': 'upload', 'time': data['timestamp'], 'fields': { - 'namespace': NAMESPACE, # Byte to Megabit 'bandwidth': data['upload']['bandwidth'] / 125000, 'bytes': data['upload']['bytes'], @@ -118,7 +119,6 @@ def format_for_influx(data): 'measurement': 'packetLoss', 'time': data['timestamp'], 'fields': { - 'namespace': NAMESPACE, 'packetLoss': pkt_loss(data) } } From fc3813f21288cf1f3869b4c24c18090ef9411ddf Mon Sep 17 00:00:00 2001 From: Leigh Phillips Date: Sat, 16 Jan 2021 23:26:39 -0800 Subject: [PATCH 13/27] Create GrafanaDash-SpeedTests.json --- GrafanaDash-SpeedTests.json | 1557 +++++++++++++++++++++++++++++++++++ 1 file changed, 1557 insertions(+) create mode 100644 GrafanaDash-SpeedTests.json diff --git a/GrafanaDash-SpeedTests.json b/GrafanaDash-SpeedTests.json new file mode 100644 index 0000000..093fc09 --- /dev/null +++ b/GrafanaDash-SpeedTests.json @@ -0,0 +1,1557 @@ +{ + "__inputs": [ + { + "name": "DS_INFLUXDB-SPEEDTESTS", + "label": "InfluxDB-speedtests", + "description": "", + "type": "datasource", + "pluginId": "influxdb", + "pluginName": "InfluxDB" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "7.3.6" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "datasource", + "id": "influxdb", + "name": "InfluxDB", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "A dashboard to display speedtest data over time.", + "editable": true, + "gnetId": 13053, + "graphTooltip": 0, + "id": null, + "iteration": 1610865850926, + "links": [], + "panels": [ + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 129, + "panels": [ + { + "datasource": "${DS_INFLUXDB-SPEEDTESTS}", + "fieldConfig": { + "defaults": { + "custom": { + "align": "center", + "filterable": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.width", + "value": 175 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "SPEEDTEST.net URL" + }, + "properties": [ + { + "id": "custom.width", + "value": 525 + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 127, + "options": { + "frameIndex": 0, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Time" + } + ] + }, + "pluginVersion": "7.3.6", + "targets": [ + { + "groupBy": [ + { + "params": [ + "speedtest_url" + ], + "type": "tag" + } + ], + "hide": false, + "measurement": "download", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT \"bandwidth\" FROM \"download\" WHERE (\"namespace\" =~ /^$namespace$/) AND $timeFilter GROUP BY \"speedtest_url\"", + "rawQuery": false, + "refId": "B", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "bandwidth" + ], + "type": "field" + }, + { + "params": [ + "Down Mbps" + ], + "type": "alias" + } + ] + ], + "tags": [ + { + "key": "namespace", + "operator": "=~", + "value": "/^$namespace$/" + } + ] + }, + { + "groupBy": [ + { + "params": [ + "speedtest_url" + ], + "type": "tag" + } + ], + "measurement": "upload", + "orderByTime": "ASC", + "policy": "default", + "refId": "C", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "bandwidth" + ], + "type": "field" + }, + { + "params": [ + "Up Mbps" + ], + "type": "alias" + } + ] + ], + "tags": [ + { + "key": "namespace", + "operator": "=~", + "value": "/^$namespace$/" + } + ] + }, + { + "groupBy": [ + { + "params": [ + "speedtest_url" + ], + "type": "tag" + } + ], + "measurement": "ping", + "orderByTime": "ASC", + "policy": "default", + "queryType": "randomWalk", + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "latency" + ], + "type": "field" + }, + { + "params": [ + "Latency ms" + ], + "type": "alias" + } + ], + [ + { + "params": [ + "jitter" + ], + "type": "field" + }, + { + "params": [ + "Jitter ms" + ], + "type": "alias" + } + ] + ], + "tags": [ + { + "key": "namespace", + "operator": "=~", + "value": "/^$namespace$/" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Panel Title", + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "Down Mbps": "", + "speedtest_url": "SPEEDTEST.net URL" + } + } + } + ], + "type": "table" + } + ], + "title": "Raw Results & Links", + "type": "row" + }, + { + "collapsed": false, + "datasource": "${DS_INFLUXDB-SPEEDTESTS}", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 108, + "panels": [], + "title": "SpeedTests", + "type": "row" + }, + { + "datasource": "${DS_INFLUXDB-SPEEDTESTS}", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "light-yellow", + "value": 700 + }, + { + "color": "green", + "value": 825 + } + ] + }, + "unit": "Mbits" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 2 + }, + "id": 116, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.3.6", + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "linear" + ], + "type": "fill" + } + ], + "measurement": "download", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "bandwidth" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "namespace", + "operator": "=~", + "value": "/^$namespace$/" + } + ] + } + ], + "timeFrom": "2d", + "timeShift": null, + "title": "Download Speed", + "type": "stat" + }, + { + "datasource": "${DS_INFLUXDB-SPEEDTESTS}", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "light-yellow", + "value": 700 + }, + { + "color": "green", + "value": 825 + } + ] + }, + "unit": "Mbits" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 2 + }, + "id": 118, + "interval": "", + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.3.6", + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "linear" + ], + "type": "fill" + } + ], + "measurement": "upload", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "bandwidth" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "namespace", + "operator": "=~", + "value": "/^$namespace$/" + } + ] + } + ], + "timeFrom": "2d", + "timeShift": null, + "title": "Upload Speed", + "type": "stat" + }, + { + "datasource": "${DS_INFLUXDB-SPEEDTESTS}", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 2 + }, + "id": 120, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.3.6", + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "linear" + ], + "type": "fill" + } + ], + "measurement": "ping", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "latency" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "namespace", + "operator": "=~", + "value": "/^$namespace$/" + } + ] + } + ], + "timeFrom": "2d", + "timeShift": null, + "title": "Response Time", + "type": "stat" + }, + { + "datasource": "${DS_INFLUXDB-SPEEDTESTS}", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 10000 + }, + { + "color": "red", + "value": 14000 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 2 + }, + "id": 122, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.3.6", + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "linear" + ], + "type": "fill" + } + ], + "measurement": "download", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "elapsed" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "namespace", + "operator": "=~", + "value": "/^$namespace$/" + } + ] + } + ], + "timeFrom": "2d", + "timeShift": null, + "title": "Test Time", + "type": "stat" + }, + { + "datasource": "${DS_INFLUXDB-SPEEDTESTS}", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 6 + }, + "id": 124, + "interval": "1h", + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.3.6", + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "previous" + ], + "type": "fill" + } + ], + "measurement": "download", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "namespace", + "operator": "=~", + "value": "/^$namespace$/" + } + ] + } + ], + "timeFrom": "2d", + "timeShift": null, + "title": "Download Size Total", + "type": "stat" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_INFLUXDB-SPEEDTESTS}", + "description": "Uses https://github.com/breadlysm/speedtest-to-influxdb", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 3, + "gridPos": { + "h": 8, + "w": 18, + "x": 6, + "y": 6 + }, + "hiddenSeries": false, + "id": 104, + "interval": "", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 0.5, + "points": true, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "Upload", + "color": "#F2CC0C" + }, + { + "alias": "Download", + "color": "#3274D9" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Download", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "download", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT mean(\"bytes\") FROM \"download\" WHERE $timeFilter GROUP BY time($__interval) fill(0)", + "rawQuery": false, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "bandwidth" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "namespace", + "operator": "=~", + "value": "/^$namespace$/" + } + ] + }, + { + "alias": "Upload", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "upload", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT mean(\"bytes\") FROM \"download\" WHERE $timeFilter GROUP BY time($__interval) fill(0)", + "rawQuery": false, + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "bandwidth" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "namespace", + "operator": "=~", + "value": "/^$namespace$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Speedtest Results", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Mbits", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "datasource": "${DS_INFLUXDB-SPEEDTESTS}", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 10 + }, + "id": 125, + "interval": "1h", + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.3.6", + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "previous" + ], + "type": "fill" + } + ], + "measurement": "upload", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "namespace", + "operator": "=~", + "value": "/^$namespace$/" + } + ] + } + ], + "timeFrom": "2d", + "timeShift": null, + "title": "Upload Size Total", + "type": "stat" + }, + { + "datasource": "${DS_INFLUXDB-SPEEDTESTS}", + "description": "Shows the avg speeds received by test site", + "fieldConfig": { + "defaults": { + "custom": { + "align": "center", + "displayMode": "color-background", + "filterable": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Mbits" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Ping" + }, + "properties": [ + { + "id": "unit", + "value": "ms" + }, + { + "id": "thresholds", + "value": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 25 + }, + { + "color": "red", + "value": 50 + } + ] + } + }, + { + "id": "custom.displayMode", + "value": "gradient-gauge" + }, + { + "id": "max", + "value": 100 + }, + { + "id": "custom.width" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Test Count" + }, + "properties": [ + { + "id": "unit", + "value": "none" + }, + { + "id": "custom.displayMode", + "value": "gradient-gauge" + }, + { + "id": "max", + "value": 25 + }, + { + "id": "custom.width", + "value": 150 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Location" + }, + "properties": [ + { + "id": "custom.displayMode", + "value": "color-text" + }, + { + "id": "custom.width", + "value": 180 + }, + { + "id": "custom.align", + "value": "left" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Download Avg" + }, + "properties": [ + { + "id": "custom.displayMode", + "value": "gradient-gauge" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 50 + }, + { + "color": "green", + "value": 100 + } + ] + } + }, + { + "id": "max" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Upload Avg" + }, + "properties": [ + { + "id": "custom.displayMode", + "value": "gradient-gauge" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 10 + }, + { + "color": "green", + "value": 50 + } + ] + } + }, + { + "id": "max" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Server" + }, + "properties": [ + { + "id": "custom.width", + "value": 300 + }, + { + "id": "custom.align", + "value": "left" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 114, + "options": { + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "Ping" + } + ] + }, + "pluginVersion": "7.3.6", + "targets": [ + { + "groupBy": [ + { + "params": [ + "server_name" + ], + "type": "tag" + }, + { + "params": [ + "server_location" + ], + "type": "tag" + }, + { + "params": [ + "server_id" + ], + "type": "tag" + } + ], + "measurement": "download", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT mean(\"bandwidth\") AS \"Download Avg\" FROM \"download\" WHERE $timeFilter GROUP BY \"server_name\", \"server_location\"", + "rawQuery": false, + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "bandwidth" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "Download Avg" + ], + "type": "alias" + } + ], + [ + { + "params": [ + "bandwidth" + ], + "type": "field" + }, + { + "params": [], + "type": "count" + }, + { + "params": [ + "Test Count" + ], + "type": "alias" + } + ] + ], + "tags": [ + { + "key": "namespace", + "operator": "=~", + "value": "/^$namespace$/" + } + ] + }, + { + "groupBy": [ + { + "params": [ + "server_id" + ], + "type": "tag" + } + ], + "measurement": "upload", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT mean(\"bandwidth\") AS \"Upload Avg\" FROM \"upload\" WHERE $timeFilter GROUP BY \"server_id\"", + "rawQuery": false, + "refId": "B", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "bandwidth" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "Upload Avg" + ], + "type": "alias" + } + ] + ], + "tags": [ + { + "key": "namespace", + "operator": "=~", + "value": "/^$namespace$/" + } + ] + }, + { + "groupBy": [ + { + "params": [ + "server_id" + ], + "type": "tag" + } + ], + "measurement": "ping", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT mean(\"bandwidth\") AS \"Upload Avg\" FROM \"upload\" WHERE $timeFilter GROUP BY \"server_id\"", + "rawQuery": false, + "refId": "C", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "latency" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "Ping" + ], + "type": "alias" + } + ] + ], + "tags": [ + { + "key": "namespace", + "operator": "=~", + "value": "/^$namespace$/" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Avg. Speed by test server", + "transformations": [ + { + "id": "seriesToColumns", + "options": { + "byField": "server_id" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Time 1": true, + "Time 2": true, + "Time 3": true, + "server_id": true + }, + "indexByName": { + "Download Avg": 3, + "Ping": 2, + "Test Count": 7, + "Time 1": 6, + "Time 2": 8, + "Time 3": 9, + "Upload Avg": 4, + "server_id": 5, + "server_location": 1, + "server_name": 0 + }, + "renameByName": { + "Time": "", + "server_location": "Location", + "server_name": "Server" + } + } + } + ], + "type": "table" + } + ], + "refresh": false, + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "current": {}, + "datasource": "${DS_INFLUXDB-SPEEDTESTS}", + "definition": "", + "error": null, + "hide": 1, + "includeAll": false, + "label": "namespace", + "multi": false, + "name": "namespace", + "options": [], + "query": "SHOW TAG VALUES FROM ping WITH Key = \"namespace\"", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "SpeedTests", + "uid": "kLXTiedGz", + "version": 9 +} From 3c1695d6a27ff40cfa84a8d8a92d1559d799a615 Mon Sep 17 00:00:00 2001 From: Leigh Phillips Date: Sat, 16 Jan 2021 23:28:32 -0800 Subject: [PATCH 14/27] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9204c89..736aa0b 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ This is a Python script that will continuously run the official Speedtest CLI ap This script will allow you to measure your internet connections speed and consistency over time. It uses env variables as configuration. It's as easy to use as telling your Docker server a 1 line command and you'll be set. Using Grafana you can start exploring this data easily. -I built a grafana dashboard for this data that can be found at https://grafana.com/grafana/dashboards/13053 +I built a Grafana dashboard which has been exported into this repo as `GrafanaDash-Speedtests.json` to import into Grafana for your convenience. ![Grafana Dashboard](https://grafana.com/api/dashboards/13053/images/8976/image) From 002a538a2c75eeb12e23db2dcc98ee705f1f071b Mon Sep 17 00:00:00 2001 From: Leigh Phillips Date: Sat, 16 Jan 2021 23:29:16 -0800 Subject: [PATCH 15/27] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 736aa0b..5c7e8e9 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ This is a Python script that will continuously run the official Speedtest CLI ap This script will allow you to measure your internet connections speed and consistency over time. It uses env variables as configuration. It's as easy to use as telling your Docker server a 1 line command and you'll be set. Using Grafana you can start exploring this data easily. -I built a Grafana dashboard which has been exported into this repo as `GrafanaDash-Speedtests.json` to import into Grafana for your convenience. +I built a Grafana dashboard which has been exported into this repo as `GrafanaDash-SpeedTests.json` to import into Grafana for your convenience. ![Grafana Dashboard](https://grafana.com/api/dashboards/13053/images/8976/image) From fd21f99132cf2179ec10652fd13caa8fb54ed2a3 Mon Sep 17 00:00:00 2001 From: Leigh Phillips Date: Sat, 16 Jan 2021 23:50:30 -0800 Subject: [PATCH 16/27] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 5c7e8e9..461edc4 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ The variables available are: - Intervals are in minutes. *Script will convert it to seconds.* - If any variables are not needed, don't declare them. Functions will operate with or without most variables. - Tags should be input without quotes. *INFLUX_DB_TAGS = isp, interface, external_ip, server_name, speedtest_url* +- NAMESPACE is used to collect data from multiple instances of the container into one database and select which you wish to view in Grafana. i.e. I have one monitoring my Starlink, the other my TELUS connection. ### Tag Options The Ookla speedtest app provides a nice set of data beyond the upload and download speed. The list is below. From 3831683c1232685152d6d0ddf055abd419f7dc2a Mon Sep 17 00:00:00 2001 From: Leigh Phillips Date: Sun, 17 Jan 2021 18:40:08 -0800 Subject: [PATCH 17/27] Update GrafanaDash-SpeedTests.json Add Leaders & Laggers section --- GrafanaDash-SpeedTests.json | 2136 ++++++++++++++++++++++++++--------- 1 file changed, 1585 insertions(+), 551 deletions(-) diff --git a/GrafanaDash-SpeedTests.json b/GrafanaDash-SpeedTests.json index 093fc09..ae905bc 100644 --- a/GrafanaDash-SpeedTests.json +++ b/GrafanaDash-SpeedTests.json @@ -59,262 +59,9 @@ "gnetId": 13053, "graphTooltip": 0, "id": null, - "iteration": 1610865850926, + "iteration": 1610934200933, "links": [], "panels": [ - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 129, - "panels": [ - { - "datasource": "${DS_INFLUXDB-SPEEDTESTS}", - "fieldConfig": { - "defaults": { - "custom": { - "align": "center", - "filterable": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Time" - }, - "properties": [ - { - "id": "custom.width", - "value": 175 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "SPEEDTEST.net URL" - }, - "properties": [ - { - "id": "custom.width", - "value": 525 - } - ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 1 - }, - "id": 127, - "options": { - "frameIndex": 0, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "Time" - } - ] - }, - "pluginVersion": "7.3.6", - "targets": [ - { - "groupBy": [ - { - "params": [ - "speedtest_url" - ], - "type": "tag" - } - ], - "hide": false, - "measurement": "download", - "orderByTime": "ASC", - "policy": "default", - "query": "SELECT \"bandwidth\" FROM \"download\" WHERE (\"namespace\" =~ /^$namespace$/) AND $timeFilter GROUP BY \"speedtest_url\"", - "rawQuery": false, - "refId": "B", - "resultFormat": "table", - "select": [ - [ - { - "params": [ - "bandwidth" - ], - "type": "field" - }, - { - "params": [ - "Down Mbps" - ], - "type": "alias" - } - ] - ], - "tags": [ - { - "key": "namespace", - "operator": "=~", - "value": "/^$namespace$/" - } - ] - }, - { - "groupBy": [ - { - "params": [ - "speedtest_url" - ], - "type": "tag" - } - ], - "measurement": "upload", - "orderByTime": "ASC", - "policy": "default", - "refId": "C", - "resultFormat": "table", - "select": [ - [ - { - "params": [ - "bandwidth" - ], - "type": "field" - }, - { - "params": [ - "Up Mbps" - ], - "type": "alias" - } - ] - ], - "tags": [ - { - "key": "namespace", - "operator": "=~", - "value": "/^$namespace$/" - } - ] - }, - { - "groupBy": [ - { - "params": [ - "speedtest_url" - ], - "type": "tag" - } - ], - "measurement": "ping", - "orderByTime": "ASC", - "policy": "default", - "queryType": "randomWalk", - "refId": "A", - "resultFormat": "table", - "select": [ - [ - { - "params": [ - "latency" - ], - "type": "field" - }, - { - "params": [ - "Latency ms" - ], - "type": "alias" - } - ], - [ - { - "params": [ - "jitter" - ], - "type": "field" - }, - { - "params": [ - "Jitter ms" - ], - "type": "alias" - } - ] - ], - "tags": [ - { - "key": "namespace", - "operator": "=~", - "value": "/^$namespace$/" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Panel Title", - "transformations": [ - { - "id": "merge", - "options": {} - }, - { - "id": "organize", - "options": { - "excludeByName": {}, - "indexByName": {}, - "renameByName": { - "Down Mbps": "", - "speedtest_url": "SPEEDTEST.net URL" - } - } - } - ], - "type": "table" - } - ], - "title": "Raw Results & Links", - "type": "row" - }, - { - "collapsed": false, - "datasource": "${DS_INFLUXDB-SPEEDTESTS}", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 1 - }, - "id": 108, - "panels": [], - "title": "SpeedTests", - "type": "row" - }, { "datasource": "${DS_INFLUXDB-SPEEDTESTS}", "fieldConfig": { @@ -346,7 +93,7 @@ "h": 4, "w": 6, "x": 0, - "y": 2 + "y": 0 }, "id": 116, "options": { @@ -444,7 +191,7 @@ "h": 4, "w": 6, "x": 6, - "y": 2 + "y": 0 }, "id": 118, "interval": "", @@ -539,7 +286,7 @@ "h": 4, "w": 6, "x": 12, - "y": 2 + "y": 0 }, "id": 120, "options": { @@ -638,7 +385,7 @@ "h": 4, "w": 6, "x": 18, - "y": 2 + "y": 0 }, "id": 122, "options": { @@ -728,7 +475,7 @@ "h": 4, "w": 6, "x": 0, - "y": 6 + "y": 4 }, "id": 124, "interval": "1h", @@ -815,7 +562,7 @@ "h": 8, "w": 18, "x": 6, - "y": 6 + "y": 4 }, "hiddenSeries": false, "id": 104, @@ -1011,7 +758,7 @@ "h": 4, "w": 6, "x": 0, - "y": 10 + "y": 8 }, "id": 125, "interval": "1h", @@ -1079,14 +826,27 @@ "title": "Upload Size Total", "type": "stat" }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 137, + "panels": [], + "title": "Leaders (Green) & Laggers (Yellow)", + "type": "row" + }, { "datasource": "${DS_INFLUXDB-SPEEDTESTS}", - "description": "Shows the avg speeds received by test site", + "description": "", "fieldConfig": { "defaults": { "custom": { "align": "center", - "displayMode": "color-background", "filterable": false }, "mappings": [], @@ -1098,290 +858,75 @@ "value": null } ] - }, - "unit": "Mbits" + } }, "overrides": [ { "matcher": { "id": "byName", - "options": "Ping" - }, - "properties": [ - { - "id": "unit", - "value": "ms" - }, - { - "id": "thresholds", - "value": { - "mode": "percentage", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "#EAB839", - "value": 25 - }, - { - "color": "red", - "value": 50 - } - ] - } - }, - { - "id": "custom.displayMode", - "value": "gradient-gauge" - }, - { - "id": "max", - "value": 100 - }, - { - "id": "custom.width" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Test Count" - }, - "properties": [ - { - "id": "unit", - "value": "none" - }, - { - "id": "custom.displayMode", - "value": "gradient-gauge" - }, - { - "id": "max", - "value": 25 - }, - { - "id": "custom.width", - "value": 150 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Location" - }, - "properties": [ - { - "id": "custom.displayMode", - "value": "color-text" - }, - { - "id": "custom.width", - "value": 180 - }, - { - "id": "custom.align", - "value": "left" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Download Avg" - }, - "properties": [ - { - "id": "custom.displayMode", - "value": "gradient-gauge" - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "yellow", - "value": 50 - }, - { - "color": "green", - "value": 100 - } - ] - } - }, - { - "id": "max" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Upload Avg" - }, - "properties": [ - { - "id": "custom.displayMode", - "value": "gradient-gauge" - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "yellow", - "value": 10 - }, - { - "color": "green", - "value": 50 - } - ] - } - }, - { - "id": "max" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Server" + "options": "Time" }, "properties": [ { "id": "custom.width", - "value": 300 + "value": 175 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "SPEEDTEST.net URL" + }, + "properties": [ + { + "id": "custom.width", + "value": 525 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Down Mbps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-green", + "mode": "fixed" + } }, { - "id": "custom.align", - "value": "left" + "id": "custom.displayMode", + "value": "color-background" } ] } ] }, "gridPos": { - "h": 8, + "h": 3, "w": 24, "x": 0, - "y": 14 + "y": 13 }, - "id": 114, + "id": 127, "options": { + "frameIndex": 0, "showHeader": true, - "sortBy": [ - { - "desc": false, - "displayName": "Ping" - } - ] + "sortBy": [] }, "pluginVersion": "7.3.6", "targets": [ { - "groupBy": [ - { - "params": [ - "server_name" - ], - "type": "tag" - }, - { - "params": [ - "server_location" - ], - "type": "tag" - }, - { - "params": [ - "server_id" - ], - "type": "tag" - } - ], + "groupBy": [], + "hide": false, "measurement": "download", "orderByTime": "ASC", "policy": "default", - "query": "SELECT mean(\"bandwidth\") AS \"Download Avg\" FROM \"download\" WHERE $timeFilter GROUP BY \"server_name\", \"server_location\"", - "rawQuery": false, - "refId": "A", - "resultFormat": "table", - "select": [ - [ - { - "params": [ - "bandwidth" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": [ - "Download Avg" - ], - "type": "alias" - } - ], - [ - { - "params": [ - "bandwidth" - ], - "type": "field" - }, - { - "params": [], - "type": "count" - }, - { - "params": [ - "Test Count" - ], - "type": "alias" - } - ] - ], - "tags": [ - { - "key": "namespace", - "operator": "=~", - "value": "/^$namespace$/" - } - ] - }, - { - "groupBy": [ - { - "params": [ - "server_id" - ], - "type": "tag" - } - ], - "measurement": "upload", - "orderByTime": "ASC", - "policy": "default", - "query": "SELECT mean(\"bandwidth\") AS \"Upload Avg\" FROM \"upload\" WHERE $timeFilter GROUP BY \"server_id\"", + "query": "SELECT max(\"bandwidth\") AS \"Down Mbps\" FROM \"download\" WHERE (\"namespace\" =~ /^$namespace$/) AND $timeFilter", "rawQuery": false, "refId": "B", "resultFormat": "table", @@ -1395,11 +940,11 @@ }, { "params": [], - "type": "mean" + "type": "max" }, { "params": [ - "Upload Avg" + "Down Mbps" ], "type": "alias" } @@ -1417,17 +962,56 @@ "groupBy": [ { "params": [ - "server_id" + "speedtest_url" ], "type": "tag" } ], + "hide": false, + "measurement": "upload", + "orderByTime": "ASC", + "policy": "default", + "refId": "C", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "bandwidth" + ], + "type": "field" + }, + { + "params": [ + "Up Mbps" + ], + "type": "alias" + } + ] + ], + "tags": [ + { + "key": "namespace", + "operator": "=~", + "value": "/^$namespace$/" + } + ] + }, + { + "groupBy": [ + { + "params": [ + "speedtest_url" + ], + "type": "tag" + } + ], + "hide": false, "measurement": "ping", "orderByTime": "ASC", "policy": "default", - "query": "SELECT mean(\"bandwidth\") AS \"Upload Avg\" FROM \"upload\" WHERE $timeFilter GROUP BY \"server_id\"", - "rawQuery": false, - "refId": "C", + "queryType": "randomWalk", + "refId": "A", "resultFormat": "table", "select": [ [ @@ -1438,12 +1022,22 @@ "type": "field" }, { - "params": [], - "type": "mean" + "params": [ + "Latency ms" + ], + "type": "alias" + } + ], + [ + { + "params": [ + "jitter" + ], + "type": "field" }, { "params": [ - "Ping" + "Jitter ms" ], "type": "alias" } @@ -1460,48 +1054,1488 @@ ], "timeFrom": null, "timeShift": null, - "title": "Avg. Speed by test server", + "title": "", "transformations": [ { - "id": "seriesToColumns", + "id": "merge", + "options": {} + }, + { + "id": "reduce", "options": { - "byField": "server_id" + "includeTimeField": true, + "mode": "reduceFields", + "reducers": [ + "firstNotNull" + ] } }, { "id": "organize", "options": { "excludeByName": { - "Time": true, - "Time 1": true, - "Time 2": true, - "Time 3": true, - "server_id": true + "Time": false }, "indexByName": { - "Download Avg": 3, - "Ping": 2, - "Test Count": 7, - "Time 1": 6, - "Time 2": 8, - "Time 3": 9, - "Upload Avg": 4, - "server_id": 5, - "server_location": 1, - "server_name": 0 + "Down Mbps": 2, + "Jitter ms": 5, + "Latency ms": 4, + "Time": 0, + "Up Mbps": 3, + "speedtest_url": 1 }, "renameByName": { - "Time": "", - "server_location": "Location", - "server_name": "Server" + "Down Mbps": "", + "speedtest_url": "SPEEDTEST.net URL" } } } ], "type": "table" + }, + { + "datasource": "${DS_INFLUXDB-SPEEDTESTS}", + "description": "", + "fieldConfig": { + "defaults": { + "custom": { + "align": "center", + "filterable": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.width", + "value": 175 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "SPEEDTEST.net URL" + }, + "properties": [ + { + "id": "custom.width", + "value": 525 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Latency ms" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-green", + "mode": "fixed" + } + }, + { + "id": "custom.displayMode", + "value": "color-background" + } + ] + } + ] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 134, + "options": { + "frameIndex": 0, + "showHeader": false, + "sortBy": [] + }, + "pluginVersion": "7.3.6", + "targets": [ + { + "groupBy": [], + "hide": false, + "measurement": "ping", + "orderByTime": "ASC", + "policy": "default", + "queryType": "randomWalk", + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "latency" + ], + "type": "field" + }, + { + "params": [], + "type": "min" + }, + { + "params": [ + "Latency ms" + ], + "type": "alias" + } + ], + [ + { + "params": [ + "jitter" + ], + "type": "field" + }, + { + "params": [ + "Jitter ms" + ], + "type": "alias" + } + ] + ], + "tags": [ + { + "key": "namespace", + "operator": "=~", + "value": "/^$namespace$/" + } + ] + }, + { + "groupBy": [ + { + "params": [ + "speedtest_url" + ], + "type": "tag" + } + ], + "hide": false, + "measurement": "download", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT max(\"bandwidth\") AS \"Down Mbps\" FROM \"download\" WHERE (\"namespace\" =~ /^$namespace$/) AND $timeFilter", + "rawQuery": false, + "refId": "B", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "bandwidth" + ], + "type": "field" + }, + { + "params": [ + "Down Mbps" + ], + "type": "alias" + } + ] + ], + "tags": [ + { + "key": "namespace", + "operator": "=~", + "value": "/^$namespace$/" + } + ] + }, + { + "groupBy": [ + { + "params": [ + "speedtest_url" + ], + "type": "tag" + } + ], + "hide": false, + "measurement": "upload", + "orderByTime": "ASC", + "policy": "default", + "refId": "C", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "bandwidth" + ], + "type": "field" + }, + { + "params": [ + "Up Mbps" + ], + "type": "alias" + } + ] + ], + "tags": [ + { + "key": "namespace", + "operator": "=~", + "value": "/^$namespace$/" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "", + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "reduce", + "options": { + "includeTimeField": true, + "mode": "reduceFields", + "reducers": [ + "firstNotNull" + ] + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": false + }, + "indexByName": { + "Down Mbps": 2, + "Jitter ms": 5, + "Latency ms": 4, + "Time": 0, + "Up Mbps": 3, + "speedtest_url": 1 + }, + "renameByName": { + "Down Mbps": "", + "speedtest_url": "SPEEDTEST.net URL" + } + } + } + ], + "type": "table" + }, + { + "datasource": "${DS_INFLUXDB-SPEEDTESTS}", + "description": "", + "fieldConfig": { + "defaults": { + "custom": { + "align": "center", + "filterable": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.width", + "value": 175 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "SPEEDTEST.net URL" + }, + "properties": [ + { + "id": "custom.width", + "value": 525 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Down Mbps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-yellow", + "mode": "fixed" + } + }, + { + "id": "custom.displayMode", + "value": "color-background" + } + ] + } + ] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 131, + "options": { + "frameIndex": 0, + "showHeader": false, + "sortBy": [] + }, + "pluginVersion": "7.3.6", + "targets": [ + { + "groupBy": [], + "hide": false, + "measurement": "download", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT max(\"bandwidth\") AS \"Down Mbps\" FROM \"download\" WHERE (\"namespace\" =~ /^$namespace$/) AND $timeFilter", + "rawQuery": false, + "refId": "B", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "bandwidth" + ], + "type": "field" + }, + { + "params": [], + "type": "min" + }, + { + "params": [ + "Down Mbps" + ], + "type": "alias" + } + ] + ], + "tags": [ + { + "key": "namespace", + "operator": "=~", + "value": "/^$namespace$/" + } + ] + }, + { + "groupBy": [ + { + "params": [ + "speedtest_url" + ], + "type": "tag" + } + ], + "hide": false, + "measurement": "upload", + "orderByTime": "ASC", + "policy": "default", + "refId": "C", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "bandwidth" + ], + "type": "field" + }, + { + "params": [ + "Up Mbps" + ], + "type": "alias" + } + ] + ], + "tags": [ + { + "key": "namespace", + "operator": "=~", + "value": "/^$namespace$/" + } + ] + }, + { + "groupBy": [ + { + "params": [ + "speedtest_url" + ], + "type": "tag" + } + ], + "hide": false, + "measurement": "ping", + "orderByTime": "ASC", + "policy": "default", + "queryType": "randomWalk", + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "latency" + ], + "type": "field" + }, + { + "params": [ + "Latency ms" + ], + "type": "alias" + } + ], + [ + { + "params": [ + "jitter" + ], + "type": "field" + }, + { + "params": [ + "Jitter ms" + ], + "type": "alias" + } + ] + ], + "tags": [ + { + "key": "namespace", + "operator": "=~", + "value": "/^$namespace$/" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "", + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "reduce", + "options": { + "includeTimeField": true, + "mode": "reduceFields", + "reducers": [ + "firstNotNull" + ] + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": false + }, + "indexByName": { + "Down Mbps": 2, + "Jitter ms": 5, + "Latency ms": 4, + "Time": 0, + "Up Mbps": 3, + "speedtest_url": 1 + }, + "renameByName": { + "Down Mbps": "", + "speedtest_url": "SPEEDTEST.net URL" + } + } + } + ], + "type": "table" + }, + { + "datasource": "${DS_INFLUXDB-SPEEDTESTS}", + "description": "", + "fieldConfig": { + "defaults": { + "custom": { + "align": "center", + "filterable": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.width", + "value": 175 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "SPEEDTEST.net URL" + }, + "properties": [ + { + "id": "custom.width", + "value": 525 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Latency ms" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-yellow", + "mode": "fixed" + } + }, + { + "id": "custom.displayMode", + "value": "color-background" + } + ] + } + ] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 135, + "options": { + "frameIndex": 0, + "showHeader": false, + "sortBy": [] + }, + "pluginVersion": "7.3.6", + "targets": [ + { + "groupBy": [], + "hide": false, + "measurement": "ping", + "orderByTime": "ASC", + "policy": "default", + "queryType": "randomWalk", + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "latency" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + }, + { + "params": [ + "Latency ms" + ], + "type": "alias" + } + ], + [ + { + "params": [ + "jitter" + ], + "type": "field" + }, + { + "params": [ + "Jitter ms" + ], + "type": "alias" + } + ] + ], + "tags": [ + { + "key": "namespace", + "operator": "=~", + "value": "/^$namespace$/" + } + ] + }, + { + "groupBy": [ + { + "params": [ + "speedtest_url" + ], + "type": "tag" + } + ], + "hide": false, + "measurement": "download", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT max(\"bandwidth\") AS \"Down Mbps\" FROM \"download\" WHERE (\"namespace\" =~ /^$namespace$/) AND $timeFilter", + "rawQuery": false, + "refId": "B", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "bandwidth" + ], + "type": "field" + }, + { + "params": [ + "Down Mbps" + ], + "type": "alias" + } + ] + ], + "tags": [ + { + "key": "namespace", + "operator": "=~", + "value": "/^$namespace$/" + } + ] + }, + { + "groupBy": [ + { + "params": [ + "speedtest_url" + ], + "type": "tag" + } + ], + "hide": false, + "measurement": "upload", + "orderByTime": "ASC", + "policy": "default", + "refId": "C", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "bandwidth" + ], + "type": "field" + }, + { + "params": [ + "Up Mbps" + ], + "type": "alias" + } + ] + ], + "tags": [ + { + "key": "namespace", + "operator": "=~", + "value": "/^$namespace$/" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "", + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "reduce", + "options": { + "includeTimeField": true, + "mode": "reduceFields", + "reducers": [ + "firstNotNull" + ] + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": false + }, + "indexByName": { + "Down Mbps": 2, + "Jitter ms": 5, + "Latency ms": 4, + "Time": 0, + "Up Mbps": 3, + "speedtest_url": 1 + }, + "renameByName": { + "Down Mbps": "", + "speedtest_url": "SPEEDTEST.net URL" + } + } + } + ], + "type": "table" + }, + { + "collapsed": true, + "datasource": "${DS_INFLUXDB-SPEEDTESTS}", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 108, + "panels": [ + { + "datasource": "${DS_INFLUXDB-SPEEDTESTS}", + "description": "Shows the avg speeds received by test site", + "fieldConfig": { + "defaults": { + "custom": { + "align": "center", + "displayMode": "color-background", + "filterable": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Mbits" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Ping" + }, + "properties": [ + { + "id": "unit", + "value": "ms" + }, + { + "id": "thresholds", + "value": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 25 + }, + { + "color": "red", + "value": 50 + } + ] + } + }, + { + "id": "custom.displayMode", + "value": "gradient-gauge" + }, + { + "id": "max", + "value": 100 + }, + { + "id": "custom.width" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Test Count" + }, + "properties": [ + { + "id": "unit", + "value": "none" + }, + { + "id": "custom.displayMode", + "value": "gradient-gauge" + }, + { + "id": "max", + "value": 25 + }, + { + "id": "custom.width", + "value": 150 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Location" + }, + "properties": [ + { + "id": "custom.displayMode", + "value": "color-text" + }, + { + "id": "custom.width", + "value": 180 + }, + { + "id": "custom.align", + "value": "left" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Download Avg" + }, + "properties": [ + { + "id": "custom.displayMode", + "value": "gradient-gauge" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 50 + }, + { + "color": "green", + "value": 100 + } + ] + } + }, + { + "id": "max" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Upload Avg" + }, + "properties": [ + { + "id": "custom.displayMode", + "value": "gradient-gauge" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 10 + }, + { + "color": "green", + "value": 50 + } + ] + } + }, + { + "id": "max" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Server" + }, + "properties": [ + { + "id": "custom.width", + "value": 300 + }, + { + "id": "custom.align", + "value": "left" + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 114, + "options": { + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "Ping" + } + ] + }, + "pluginVersion": "7.3.6", + "targets": [ + { + "groupBy": [ + { + "params": [ + "server_name" + ], + "type": "tag" + }, + { + "params": [ + "server_location" + ], + "type": "tag" + }, + { + "params": [ + "server_id" + ], + "type": "tag" + } + ], + "measurement": "download", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT mean(\"bandwidth\") AS \"Download Avg\" FROM \"download\" WHERE $timeFilter GROUP BY \"server_name\", \"server_location\"", + "rawQuery": false, + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "bandwidth" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "Download Avg" + ], + "type": "alias" + } + ], + [ + { + "params": [ + "bandwidth" + ], + "type": "field" + }, + { + "params": [], + "type": "count" + }, + { + "params": [ + "Test Count" + ], + "type": "alias" + } + ] + ], + "tags": [ + { + "key": "namespace", + "operator": "=~", + "value": "/^$namespace$/" + } + ] + }, + { + "groupBy": [ + { + "params": [ + "server_id" + ], + "type": "tag" + } + ], + "measurement": "upload", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT mean(\"bandwidth\") AS \"Upload Avg\" FROM \"upload\" WHERE $timeFilter GROUP BY \"server_id\"", + "rawQuery": false, + "refId": "B", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "bandwidth" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "Upload Avg" + ], + "type": "alias" + } + ] + ], + "tags": [ + { + "key": "namespace", + "operator": "=~", + "value": "/^$namespace$/" + } + ] + }, + { + "groupBy": [ + { + "params": [ + "server_id" + ], + "type": "tag" + } + ], + "measurement": "ping", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT mean(\"bandwidth\") AS \"Upload Avg\" FROM \"upload\" WHERE $timeFilter GROUP BY \"server_id\"", + "rawQuery": false, + "refId": "C", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "latency" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "Ping" + ], + "type": "alias" + } + ] + ], + "tags": [ + { + "key": "namespace", + "operator": "=~", + "value": "/^$namespace$/" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "", + "transformations": [ + { + "id": "seriesToColumns", + "options": { + "byField": "server_id" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Time 1": true, + "Time 2": true, + "Time 3": true, + "server_id": true + }, + "indexByName": { + "Download Avg": 3, + "Ping": 2, + "Test Count": 7, + "Time 1": 6, + "Time 2": 8, + "Time 3": 9, + "Upload Avg": 4, + "server_id": 5, + "server_location": 1, + "server_name": 0 + }, + "renameByName": { + "Time": "", + "server_location": "Location", + "server_name": "Server" + } + } + } + ], + "type": "table" + } + ], + "title": "Averages by Test Server", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 129, + "panels": [ + { + "datasource": "${DS_INFLUXDB-SPEEDTESTS}", + "fieldConfig": { + "defaults": { + "custom": { + "align": "center", + "filterable": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.width", + "value": 175 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "SPEEDTEST.net URL" + }, + "properties": [ + { + "id": "custom.width", + "value": 525 + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 36 + }, + "id": 130, + "options": { + "frameIndex": 0, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Down Mbps" + } + ] + }, + "pluginVersion": "7.3.6", + "targets": [ + { + "groupBy": [ + { + "params": [ + "speedtest_url" + ], + "type": "tag" + } + ], + "hide": false, + "measurement": "download", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT max(\"bandwidth\") AS \"Down Mbps\" FROM \"download\" WHERE (\"namespace\" =~ /^$namespace$/) AND $timeFilter", + "rawQuery": false, + "refId": "B", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "bandwidth" + ], + "type": "field" + }, + { + "params": [ + "Down Mbps" + ], + "type": "alias" + } + ] + ], + "tags": [ + { + "key": "namespace", + "operator": "=~", + "value": "/^$namespace$/" + } + ] + }, + { + "groupBy": [ + { + "params": [ + "speedtest_url" + ], + "type": "tag" + } + ], + "hide": false, + "measurement": "upload", + "orderByTime": "ASC", + "policy": "default", + "refId": "C", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "bandwidth" + ], + "type": "field" + }, + { + "params": [ + "Up Mbps" + ], + "type": "alias" + } + ] + ], + "tags": [ + { + "key": "namespace", + "operator": "=~", + "value": "/^$namespace$/" + } + ] + }, + { + "groupBy": [ + { + "params": [ + "speedtest_url" + ], + "type": "tag" + } + ], + "hide": false, + "measurement": "ping", + "orderByTime": "ASC", + "policy": "default", + "queryType": "randomWalk", + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "latency" + ], + "type": "field" + }, + { + "params": [ + "Latency ms" + ], + "type": "alias" + } + ], + [ + { + "params": [ + "jitter" + ], + "type": "field" + }, + { + "params": [ + "Jitter ms" + ], + "type": "alias" + } + ] + ], + "tags": [ + { + "key": "namespace", + "operator": "=~", + "value": "/^$namespace$/" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "All Raw", + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": false + }, + "indexByName": { + "Down Mbps": 2, + "Jitter ms": 5, + "Latency ms": 4, + "Time": 0, + "Up Mbps": 3, + "speedtest_url": 1 + }, + "renameByName": { + "Down Mbps": "", + "speedtest_url": "SPEEDTEST.net URL" + } + } + } + ], + "type": "table" + } + ], + "title": "Raw Results & Links", + "type": "row" } ], - "refresh": false, + "refresh": "1m", "schemaVersion": 26, "style": "dark", "tags": [], @@ -1533,7 +2567,7 @@ ] }, "time": { - "from": "now-6h", + "from": "now-2d", "to": "now" }, "timepicker": { @@ -1553,5 +2587,5 @@ "timezone": "", "title": "SpeedTests", "uid": "kLXTiedGz", - "version": 9 + "version": 16 } From 2d27de83aef2a79138ae41ef6ba68e52570fcdfc Mon Sep 17 00:00:00 2001 From: Leigh Phillips Date: Tue, 19 Jan 2021 23:04:06 -0800 Subject: [PATCH 18/27] Update main.py Add uptime ping test (separate interval), unify speed tests under new measurement. --- main.py | 120 +++++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 88 insertions(+), 32 deletions(-) diff --git a/main.py b/main.py index 99fd4d2..3eb60e4 100755 --- a/main.py +++ b/main.py @@ -1,7 +1,9 @@ +import os import time import json +import datetime import subprocess -import os +from pythonping import ping from influxdb import InfluxDBClient # InfluxDB Settings @@ -12,6 +14,7 @@ DB_USER = os.getenv('INFLUX_DB_USER', '') DB_PASSWORD = os.getenv('INFLUX_DB_PASSWORD', '') DB_DATABASE = os.getenv('INFLUX_DB_DATABASE', 'speedtests') DB_TAGS = os.getenv('INFLUX_DB_TAGS', None) +PING_TARGETS = os.getenv('PING_TARGETS', '1.1.1.1, 8.8.8.8') # Speedtest Settings # Time between tests (in minutes, converts to seconds). @@ -20,6 +23,8 @@ TEST_INTERVAL = int(os.getenv('SPEEDTEST_INTERVAL', '5')) * 60 TEST_FAIL_INTERVAL = int(os.getenv('SPEEDTEST_FAIL_INTERVAL', '5')) * 60 # Specific server ID SERVER_ID = os.getenv('SPEEDTEST_SERVER_ID', '') +# Time between ping tests (in seconds). +PING_INTERVAL = int(os.getenv('PING_INTERVAL', '5')) influxdb_client = InfluxDBClient( DB_ADDRESS, DB_PORT, DB_USER, DB_PASSWORD, None) @@ -50,7 +55,7 @@ def tag_selection(data): # tag_switch takes in _data and attaches CLIoutput to more readable ids tag_switch = { 'namespace': NAMESPACE, - 'isp': data['isp'], + 'isp': data['isp'], 'interface': data['interface']['name'], 'internal_ip': data['interface']['internalIp'], 'interface_mac': data['interface']['macAddr'], @@ -66,7 +71,7 @@ def tag_selection(data): 'speedtest_id': data['result']['id'], 'speedtest_url': data['result']['url'] } - + if tags is None: tags = 'namespace' elif '*' in tags: @@ -79,12 +84,12 @@ def tag_selection(data): # split the tag string, strip and add selected tags to {options} with corresponding tag_switch data tag = tag.strip() options[tag] = tag_switch[tag] - + return options def format_for_influx(data): - + # There is additional data in the speedtest-cli output but it is likely not necessary to store. influx_data = [ { @@ -121,45 +126,96 @@ def format_for_influx(data): 'fields': { 'packetLoss': pkt_loss(data) } + }, + { + 'measurement': 'speeds', + 'time': data['timestamp'], + 'fields': { + 'jitter': data['ping']['jitter'], + 'latency': data['ping']['latency'], + 'packetLoss': pkt_loss(data), + # Byte to Megabit + 'bandwidth_down': data['download']['bandwidth'] / 125000, + 'bytes_down': data['download']['bytes'], + 'elapsed_down': data['download']['elapsed'], + # Byte to Megabit + 'bandwidth_up': data['upload']['bandwidth'] / 125000, + 'bytes_up': data['upload']['bytes'], + 'elapsed_up': data['upload']['elapsed'] + } } ] tags = tag_selection(data) - if tags is None: - return influx_data - else: + if tags is not None: for measurement in influx_data: measurement['tags'] = tags - return influx_data + return influx_data + + +def speedtest(): + if not SERVER_ID: + speedtest = subprocess.run( + ["speedtest", "--accept-license", "--accept-gdpr", "-f", "json"], capture_output=True) + print("Automatic server choice") + else: + speedtest = subprocess.run( + ["speedtest", "--accept-license", "--accept-gdpr", "-f", "json", "--server-id=" + SERVER_ID], capture_output=True) + print("Manual server choice : ID = " + SERVER_ID) + + if speedtest.returncode == 0: # Speedtest was successful. + print("Speedtest Successful :") + data_json = json.loads(speedtest.stdout) + print("time: " + str(data_json['timestamp']) + " - ping: " + str(data_json['ping']['latency']) + " ms - download: " + str(data_json['download']['bandwidth']/125000) + " Mb/s - upload: " + str(data_json['upload']['bandwidth'] / 125000) + " Mb/s - isp: " + data_json['isp'] + " - ext. IP: " + data_json['interface']['externalIp'] + " - server id: " + str(data_json['server']['id']) + " (" + data_json['server']['name'] + " @ " + data_json['server']['location'] + ")") + data = format_for_influx(data_json) + if influxdb_client.write_points(data) == True: + print("Data written to DB successfully") + else: # Speedtest failed. + print("Speedtest Failed :") + print(speedtest.stderr) + print(speedtest.stdout) +# time.sleep(TEST_FAIL_INTERVAL) + + +def pingtest(): + timestamp = datetime.datetime.utcnow() + for target in PING_TARGETS.split(','): + target = target.strip() + pingtest = ping(target, verbose=False, timeout=1, count=1, size=128) + data = [ + { + 'measurement': 'pings', + 'time': timestamp, + 'tags': { + 'target' : target + }, + 'fields': { + 'success' : pingtest.success(), + 'rtt': pingtest.rtt_avg_ms + } + } + ] + if influxdb_client.write_points(data) == True: + print("Ping data written to DB successfully") + else: # Speedtest failed. + print("Ping Failed.") def main(): init_db() # Setup the database if it does not already exist. + loopcount = 1 while (1): # Run a Speedtest and send the results to influxDB indefinitely. - server_id = SERVER_ID - if not server_id: - speedtest = subprocess.run( - ["speedtest", "--accept-license", "--accept-gdpr", "-f", "json"], capture_output=True) - print("Automatic server choice") - else: - speedtest = subprocess.run( - ["speedtest", "--accept-license", "--accept-gdpr", "-f", "json", "--server-id=" + SERVER_ID], capture_output=True) - print("Manual server choice : ID = " + SERVER_ID) - - if speedtest.returncode == 0: # Speedtest was successful. - print("Speedtest Successful :") - data_json = json.loads(speedtest.stdout) - print("time: " + str(data_json['timestamp']) + " - ping: " + str(data_json['ping']['latency']) + " ms - download: " + str(data_json['download']['bandwidth']/125000) + " Mb/s - upload: " + str(data_json['upload']['bandwidth'] / 125000) + " Mb/s - isp: " + data_json['isp'] + " - ext. IP: " + data_json['interface']['externalIp'] + " - server id: " + str(data_json['server']['id']) + " (" + data_json['server']['name'] + " @ " + data_json['server']['location'] + ")") - data = format_for_influx(data_json) - if influxdb_client.write_points(data) == True: - print("Data written to DB successfully") - time.sleep(TEST_INTERVAL) - else: # Speedtest failed. - print("Speedtest Failed :") - print(speedtest.stderr) - print(speedtest.stdout) - time.sleep(TEST_FAIL_INTERVAL) + if loopcount % PING_INTERVAL == 0: + pingtest() + if loopcount % TEST_INTERVAL == 0: + speedtest() + + if loopcount % ( PING_INTERVAL * TEST_INTERVAL ) == 0: + loopcount = 0 + + time.sleep(1) + loopcount += 1 if __name__ == '__main__': print('Speedtest CLI data logger to InfluxDB started...') From 7dc293bc166592859456b63f857e5cd9d8e01df4 Mon Sep 17 00:00:00 2001 From: Leigh Phillips Date: Tue, 19 Jan 2021 23:06:51 -0800 Subject: [PATCH 19/27] Update README.md --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 461edc4..d6cbb5b 100644 --- a/README.md +++ b/README.md @@ -22,9 +22,11 @@ The variables available are: - INFLUX_DB_PASSWORD = default - {blank} - INFLUX_DB_DATABASE = default - speedtests - INFLUX_DB_TAGS = default - None * See below for options, '*' widcard for all * -- SPEEDTEST_INTERVAL = default - 5 -- SPEEDTEST_FAIL_INTERVAL = deafult - 5 +- SPEEDTEST_INTERVAL = default - 5 (minutes) +- SPEEDTEST_FAIL_INTERVAL = deafult - 5 (now deprecated) - SPEEDTEST_SERVER_ID = default - {blank} * id from https://c.speedtest.net/speedtest-servers-static.php * +- PING_INTERVAL = default - 5 (seconds) +- PING_TARGETS = deafault - 1.1.1.1, 8.8.8.8 (csv of hosts to ping) ### Variable Notes - Intervals are in minutes. *Script will convert it to seconds.* From e33443ec0afcc2ee62eb603c2850e000462c91d5 Mon Sep 17 00:00:00 2001 From: Leigh Phillips Date: Tue, 19 Jan 2021 23:18:41 -0800 Subject: [PATCH 20/27] Update Dockerfile Add pip3 pythonping --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 821207e..e922f42 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,7 @@ apt-get update && \ apt-get -q -y install --no-install-recommends apt-utils gnupg1 apt-transport-https dirmngr && \ \ # Install Python packages -pip3 install influxdb && \ +pip3 install pythonping influxdb && \ \ # Clean up apt-get -q -y autoremove && apt-get -q -y clean && \ From 74f125608435d724f357c4c2530e1f03f09e0435 Mon Sep 17 00:00:00 2001 From: Leigh Phillips Date: Tue, 19 Jan 2021 23:24:47 -0800 Subject: [PATCH 21/27] Update main.py Add namespace to pings. --- main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/main.py b/main.py index 3eb60e4..266c160 100755 --- a/main.py +++ b/main.py @@ -187,6 +187,7 @@ def pingtest(): 'measurement': 'pings', 'time': timestamp, 'tags': { + 'namespace': NAMESPACE, 'target' : target }, 'fields': { From 1254627483644ce3ae9364138d7dc3b97d90d1a8 Mon Sep 17 00:00:00 2001 From: Leigh Phillips Date: Wed, 20 Jan 2021 00:02:54 -0800 Subject: [PATCH 22/27] Update main.py Change ping.success from True/False to 1/0 --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index 266c160..50fc1c9 100755 --- a/main.py +++ b/main.py @@ -191,7 +191,7 @@ def pingtest(): 'target' : target }, 'fields': { - 'success' : pingtest.success(), + 'success' : int(pingtest.success()), 'rtt': pingtest.rtt_avg_ms } } From 85505c3087d6d0d80d90c6e4ba90f7c645fce1b3 Mon Sep 17 00:00:00 2001 From: Leigh Phillips Date: Wed, 20 Jan 2021 15:53:13 -0800 Subject: [PATCH 23/27] Update entrypoint.sh Do not reinstall speedtest if already installed. --- entrypoint.sh | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index 6bd11aa..9198112 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -4,10 +4,13 @@ printenv >> /etc/environment ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone # Install speedtest-cli -apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 379CE192D401AB61 -echo "deb https://ookla.bintray.com/debian buster main" | tee /etc/apt/sources.list.d/speedtest.list -apt-get update && apt-get -q -y install speedtest -#apt-get -q -y autoremove && apt-get -q -y clean -#rm -rf /var/lib/apt/lists/* +if [ ! -e /usr/bin/speedtest ] +then + apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 379CE192D401AB61 + echo "deb https://ookla.bintray.com/debian buster main" | tee /etc/apt/sources.list.d/speedtest.list + apt-get update && apt-get -q -y install speedtest + apt-get -q -y autoremove && apt-get -q -y clean + rm -rf /var/lib/apt/lists/* +fi exec /usr/local/bin/python3 $@ From add7337b1d532ceaa2b6a39bc10d3d24964ee604 Mon Sep 17 00:00:00 2001 From: Leigh Phillips Date: Wed, 20 Jan 2021 15:54:19 -0800 Subject: [PATCH 24/27] Update main.py Go multithreaded. --- main.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/main.py b/main.py index 50fc1c9..cb8a782 100755 --- a/main.py +++ b/main.py @@ -5,6 +5,7 @@ import datetime import subprocess from pythonping import ping from influxdb import InfluxDBClient +from multiprocessing import Process # InfluxDB Settings NAMESPACE = os.getenv('NAMESPACE', 'None') @@ -191,8 +192,8 @@ def pingtest(): 'target' : target }, 'fields': { - 'success' : int(pingtest.success()), - 'rtt': pingtest.rtt_avg_ms + 'success' : int(pingtest._responses[0].error_message is None), + 'rtt': float(0 if pingtest._responses[0].error_message is not None else pingtest.rtt_avg_ms) } } ] @@ -202,15 +203,24 @@ def pingtest(): print("Ping Failed.") def main(): + pPing = Process(target=pingtest) + pSpeed = Process(target=speedtest) + init_db() # Setup the database if it does not already exist. - loopcount = 1 + loopcount = 0 while (1): # Run a Speedtest and send the results to influxDB indefinitely. - if loopcount % PING_INTERVAL == 0: - pingtest() + if loopcount == 0 or loopcount % PING_INTERVAL == 0: + if pPing.is_alive(): + pPing.terminate() + pPing = Process(target=pingtest) + pPing.start() - if loopcount % TEST_INTERVAL == 0: - speedtest() + if loopcount == 0 or loopcount % TEST_INTERVAL == 0: + if pSpeed.is_alive(): + pSpeed.terminate() + pSpeed = Process(target=speedtest) + pSpeed.start() if loopcount % ( PING_INTERVAL * TEST_INTERVAL ) == 0: loopcount = 0 From c1716c0301bbdbfbc3aa88ac16b689a88c024092 Mon Sep 17 00:00:00 2001 From: Leigh Phillips Date: Wed, 20 Jan 2021 22:04:18 -0800 Subject: [PATCH 25/27] Update GrafanaDash-SpeedTests.json Now with High Frequency Ping results and uptime! --- GrafanaDash-SpeedTests.json | 2065 ++++++++++++++++++++--------------- 1 file changed, 1211 insertions(+), 854 deletions(-) diff --git a/GrafanaDash-SpeedTests.json b/GrafanaDash-SpeedTests.json index ae905bc..901fdd1 100644 --- a/GrafanaDash-SpeedTests.json +++ b/GrafanaDash-SpeedTests.json @@ -59,13 +59,17 @@ "gnetId": 13053, "graphTooltip": 0, "id": null, - "iteration": 1610934200933, + "iteration": 1611206261586, "links": [], "panels": [ { "datasource": "${DS_INFLUXDB-SPEEDTESTS}", "fieldConfig": { "defaults": { + "color": { + "fixedColor": "semi-dark-blue", + "mode": "fixed" + }, "custom": {}, "mappings": [], "thresholds": { @@ -90,12 +94,13 @@ "overrides": [] }, "gridPos": { - "h": 4, - "w": 6, + "h": 2, + "w": 3, "x": 0, "y": 0 }, "id": 116, + "interval": "5m", "options": { "colorMode": "value", "graphMode": "area", @@ -119,12 +124,6 @@ "$__interval" ], "type": "time" - }, - { - "params": [ - "linear" - ], - "type": "fill" } ], "measurement": "download", @@ -155,392 +154,9 @@ ] } ], - "timeFrom": "2d", + "timeFrom": null, "timeShift": null, - "title": "Download Speed", - "type": "stat" - }, - { - "datasource": "${DS_INFLUXDB-SPEEDTESTS}", - "fieldConfig": { - "defaults": { - "custom": {}, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "light-yellow", - "value": 700 - }, - { - "color": "green", - "value": 825 - } - ] - }, - "unit": "Mbits" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 6, - "y": 0 - }, - "id": 118, - "interval": "", - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.3.6", - "targets": [ - { - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "linear" - ], - "type": "fill" - } - ], - "measurement": "upload", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "bandwidth" - ], - "type": "field" - }, - { - "params": [], - "type": "last" - } - ] - ], - "tags": [ - { - "key": "namespace", - "operator": "=~", - "value": "/^$namespace$/" - } - ] - } - ], - "timeFrom": "2d", - "timeShift": null, - "title": "Upload Speed", - "type": "stat" - }, - { - "datasource": "${DS_INFLUXDB-SPEEDTESTS}", - "fieldConfig": { - "defaults": { - "custom": {}, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 12, - "y": 0 - }, - "id": 120, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.3.6", - "targets": [ - { - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "linear" - ], - "type": "fill" - } - ], - "measurement": "ping", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "latency" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [ - { - "key": "namespace", - "operator": "=~", - "value": "/^$namespace$/" - } - ] - } - ], - "timeFrom": "2d", - "timeShift": null, - "title": "Response Time", - "type": "stat" - }, - { - "datasource": "${DS_INFLUXDB-SPEEDTESTS}", - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": 2, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 10000 - }, - { - "color": "red", - "value": 14000 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 18, - "y": 0 - }, - "id": 122, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.3.6", - "targets": [ - { - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "linear" - ], - "type": "fill" - } - ], - "measurement": "download", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "elapsed" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [ - { - "key": "namespace", - "operator": "=~", - "value": "/^$namespace$/" - } - ] - } - ], - "timeFrom": "2d", - "timeShift": null, - "title": "Test Time", - "type": "stat" - }, - { - "datasource": "${DS_INFLUXDB-SPEEDTESTS}", - "fieldConfig": { - "defaults": { - "custom": {}, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 4 - }, - "id": 124, - "interval": "1h", - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "sum" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.3.6", - "targets": [ - { - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "previous" - ], - "type": "fill" - } - ], - "measurement": "download", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "bytes" - ], - "type": "field" - }, - { - "params": [], - "type": "last" - } - ] - ], - "tags": [ - { - "key": "namespace", - "operator": "=~", - "value": "/^$namespace$/" - } - ] - } - ], - "timeFrom": "2d", - "timeShift": null, - "title": "Download Size Total", + "title": "Avg Download Speed", "type": "stat" }, { @@ -559,10 +175,10 @@ "fill": 1, "fillGradient": 3, "gridPos": { - "h": 8, + "h": 7, "w": 18, - "x": 6, - "y": 4 + "x": 3, + "y": 0 }, "hiddenSeries": false, "id": 104, @@ -592,7 +208,7 @@ "seriesOverrides": [ { "alias": "Upload", - "color": "#F2CC0C" + "color": "#A352CC" }, { "alias": "Download", @@ -611,12 +227,6 @@ "$__interval" ], "type": "time" - }, - { - "params": [ - "none" - ], - "type": "fill" } ], "measurement": "download", @@ -656,12 +266,6 @@ "$__interval" ], "type": "time" - }, - { - "params": [ - "none" - ], - "type": "fill" } ], "measurement": "upload", @@ -735,6 +339,201 @@ "alignLevel": null } }, + { + "datasource": "${DS_INFLUXDB-SPEEDTESTS}", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "semi-dark-yellow", + "mode": "fixed" + }, + "custom": {}, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 10000 + }, + { + "color": "red", + "value": 14000 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 21, + "y": 0 + }, + "id": 122, + "interval": "5m", + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.3.6", + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + } + ], + "measurement": "download", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "elapsed" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "namespace", + "operator": "=~", + "value": "/^$namespace$/" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Avg Test Time", + "type": "stat" + }, + { + "datasource": "${DS_INFLUXDB-SPEEDTESTS}", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "semi-dark-purple", + "mode": "fixed" + }, + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "light-yellow", + "value": 700 + }, + { + "color": "green", + "value": 825 + } + ] + }, + "unit": "Mbits" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 0, + "y": 2 + }, + "id": 118, + "interval": "5m", + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.3.6", + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + } + ], + "measurement": "upload", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "bandwidth" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "namespace", + "operator": "=~", + "value": "/^$namespace$/" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Avg Upload Speed", + "type": "stat" + }, { "datasource": "${DS_INFLUXDB-SPEEDTESTS}", "fieldConfig": { @@ -755,13 +554,13 @@ "overrides": [] }, "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 8 + "h": 2, + "w": 3, + "x": 21, + "y": 3 }, - "id": 125, - "interval": "1h", + "id": 124, + "interval": "5m", "options": { "colorMode": "value", "graphMode": "area", @@ -785,12 +584,189 @@ "$__interval" ], "type": "time" - }, + } + ], + "measurement": "download", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "namespace", + "operator": "=~", + "value": "/^$namespace$/" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Total Test Downloaded", + "type": "stat" + }, + { + "datasource": "${DS_INFLUXDB-SPEEDTESTS}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "semi-dark-yellow", + "mode": "fixed" + }, + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 90 + }, + { + "color": "red", + "value": 150 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 0, + "y": 4 + }, + "id": 120, + "interval": "5m", + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.3.6", + "targets": [ + { + "groupBy": [ { "params": [ - "previous" + "$__interval" ], - "type": "fill" + "type": "time" + } + ], + "measurement": "ping", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "latency" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "namespace", + "operator": "=~", + "value": "/^$namespace$/" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Avg Test Latency", + "type": "stat" + }, + { + "datasource": "${DS_INFLUXDB-SPEEDTESTS}", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 21, + "y": 5 + }, + "id": 125, + "interval": "5m", + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.3.6", + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" } ], "measurement": "upload", @@ -821,9 +797,391 @@ ] } ], - "timeFrom": "2d", + "timeFrom": null, "timeShift": null, - "title": "Upload Size Total", + "title": "Total Test Uploaded", + "type": "stat" + }, + { + "datasource": "${DS_INFLUXDB-SPEEDTESTS}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "dark-green", + "mode": "fixed" + }, + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 0, + "y": 7 + }, + "id": 139, + "interval": "", + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.3.6", + "targets": [ + { + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + } + ], + "measurement": "pings", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "success" + ], + "type": "field" + }, + { + "params": [], + "type": "min" + } + ] + ], + "tags": [ + { + "key": "namespace", + "operator": "=~", + "value": "/^$namespace$/" + } + ], + "tz": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Uptime", + "type": "stat" + }, + { + "aliasColors": { + "Latency": "semi-dark-yellow", + "Uptime": "dark-green", + "pings.min": "light-green", + "pings.sucess": "dark-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_INFLUXDB-SPEEDTESTS}", + "description": "", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 3, + "gridPos": { + "h": 6, + "w": 21, + "x": 3, + "y": 7 + }, + "hiddenSeries": false, + "id": 138, + "interval": "", + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": 180, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 0.5, + "points": true, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "pings.min", + "yaxis": 2 + }, + { + "alias": "pings.sucess", + "yaxis": 2 + }, + { + "alias": "Uptime", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Latency", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + } + ], + "measurement": "pings", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT mean(\"bytes\") FROM \"download\" WHERE $timeFilter GROUP BY time($__interval) fill(0)", + "rawQuery": false, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "rtt" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "namespace", + "operator": "=~", + "value": "/^$namespace$/" + } + ] + }, + { + "alias": "Uptime", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + } + ], + "measurement": "pings", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT mean(\"bytes\") FROM \"download\" WHERE $timeFilter GROUP BY time($__interval) fill(0)", + "rawQuery": false, + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "success" + ], + "type": "field" + }, + { + "params": [], + "type": "min" + } + ] + ], + "tags": [ + { + "key": "namespace", + "operator": "=~", + "value": "/^$namespace$/" + } + ], + "tz": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "High Frequency Ping Results", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 2, + "max": "256", + "min": "4", + "show": true + }, + { + "decimals": null, + "format": "percentunit", + "label": "", + "logBase": 1, + "max": "1", + "min": "0", + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "datasource": "${DS_INFLUXDB-SPEEDTESTS}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "semi-dark-yellow", + "mode": "fixed" + }, + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 90 + }, + { + "color": "red", + "value": 150 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 0, + "y": 10 + }, + "id": 140, + "interval": "", + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.3.6", + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + } + ], + "measurement": "pings", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "rtt" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "namespace", + "operator": "=~", + "value": "/^$namespace$/" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Avg Ping Latency", "type": "stat" }, { @@ -833,7 +1191,7 @@ "h": 1, "w": 24, "x": 0, - "y": 12 + "y": 13 }, "id": 137, "panels": [], @@ -910,7 +1268,7 @@ "h": 3, "w": 24, "x": 0, - "y": 13 + "y": 14 }, "id": 127, "options": { @@ -1163,7 +1521,7 @@ "h": 2, "w": 24, "x": 0, - "y": 16 + "y": 17 }, "id": 134, "options": { @@ -1416,7 +1774,7 @@ "h": 2, "w": 24, "x": 0, - "y": 18 + "y": 19 }, "id": 131, "options": { @@ -1669,7 +2027,7 @@ "h": 2, "w": 24, "x": 0, - "y": 20 + "y": 21 }, "id": 135, "options": { @@ -1853,441 +2211,440 @@ "type": "table" }, { - "collapsed": true, + "collapsed": false, "datasource": "${DS_INFLUXDB-SPEEDTESTS}", "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 22 + "y": 23 }, "id": 108, - "panels": [ - { - "datasource": "${DS_INFLUXDB-SPEEDTESTS}", - "description": "Shows the avg speeds received by test site", - "fieldConfig": { - "defaults": { - "custom": { - "align": "center", - "displayMode": "color-background", - "filterable": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "Mbits" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Ping" - }, - "properties": [ - { - "id": "unit", - "value": "ms" - }, - { - "id": "thresholds", - "value": { - "mode": "percentage", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "#EAB839", - "value": 25 - }, - { - "color": "red", - "value": 50 - } - ] - } - }, - { - "id": "custom.displayMode", - "value": "gradient-gauge" - }, - { - "id": "max", - "value": 100 - }, - { - "id": "custom.width" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Test Count" - }, - "properties": [ - { - "id": "unit", - "value": "none" - }, - { - "id": "custom.displayMode", - "value": "gradient-gauge" - }, - { - "id": "max", - "value": 25 - }, - { - "id": "custom.width", - "value": 150 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Location" - }, - "properties": [ - { - "id": "custom.displayMode", - "value": "color-text" - }, - { - "id": "custom.width", - "value": 180 - }, - { - "id": "custom.align", - "value": "left" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Download Avg" - }, - "properties": [ - { - "id": "custom.displayMode", - "value": "gradient-gauge" - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "yellow", - "value": 50 - }, - { - "color": "green", - "value": 100 - } - ] - } - }, - { - "id": "max" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Upload Avg" - }, - "properties": [ - { - "id": "custom.displayMode", - "value": "gradient-gauge" - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "yellow", - "value": 10 - }, - { - "color": "green", - "value": 50 - } - ] - } - }, - { - "id": "max" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Server" - }, - "properties": [ - { - "id": "custom.width", - "value": 300 - }, - { - "id": "custom.align", - "value": "left" - } - ] - } - ] - }, - "gridPos": { - "h": 12, - "w": 24, - "x": 0, - "y": 23 - }, - "id": 114, - "options": { - "showHeader": true, - "sortBy": [ - { - "desc": false, - "displayName": "Ping" - } - ] - }, - "pluginVersion": "7.3.6", - "targets": [ - { - "groupBy": [ - { - "params": [ - "server_name" - ], - "type": "tag" - }, - { - "params": [ - "server_location" - ], - "type": "tag" - }, - { - "params": [ - "server_id" - ], - "type": "tag" - } - ], - "measurement": "download", - "orderByTime": "ASC", - "policy": "default", - "query": "SELECT mean(\"bandwidth\") AS \"Download Avg\" FROM \"download\" WHERE $timeFilter GROUP BY \"server_name\", \"server_location\"", - "rawQuery": false, - "refId": "A", - "resultFormat": "table", - "select": [ - [ - { - "params": [ - "bandwidth" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": [ - "Download Avg" - ], - "type": "alias" - } - ], - [ - { - "params": [ - "bandwidth" - ], - "type": "field" - }, - { - "params": [], - "type": "count" - }, - { - "params": [ - "Test Count" - ], - "type": "alias" - } - ] - ], - "tags": [ - { - "key": "namespace", - "operator": "=~", - "value": "/^$namespace$/" - } - ] - }, - { - "groupBy": [ - { - "params": [ - "server_id" - ], - "type": "tag" - } - ], - "measurement": "upload", - "orderByTime": "ASC", - "policy": "default", - "query": "SELECT mean(\"bandwidth\") AS \"Upload Avg\" FROM \"upload\" WHERE $timeFilter GROUP BY \"server_id\"", - "rawQuery": false, - "refId": "B", - "resultFormat": "table", - "select": [ - [ - { - "params": [ - "bandwidth" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": [ - "Upload Avg" - ], - "type": "alias" - } - ] - ], - "tags": [ - { - "key": "namespace", - "operator": "=~", - "value": "/^$namespace$/" - } - ] - }, - { - "groupBy": [ - { - "params": [ - "server_id" - ], - "type": "tag" - } - ], - "measurement": "ping", - "orderByTime": "ASC", - "policy": "default", - "query": "SELECT mean(\"bandwidth\") AS \"Upload Avg\" FROM \"upload\" WHERE $timeFilter GROUP BY \"server_id\"", - "rawQuery": false, - "refId": "C", - "resultFormat": "table", - "select": [ - [ - { - "params": [ - "latency" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": [ - "Ping" - ], - "type": "alias" - } - ] - ], - "tags": [ - { - "key": "namespace", - "operator": "=~", - "value": "/^$namespace$/" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "", - "transformations": [ - { - "id": "seriesToColumns", - "options": { - "byField": "server_id" - } - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "Time 1": true, - "Time 2": true, - "Time 3": true, - "server_id": true - }, - "indexByName": { - "Download Avg": 3, - "Ping": 2, - "Test Count": 7, - "Time 1": 6, - "Time 2": 8, - "Time 3": 9, - "Upload Avg": 4, - "server_id": 5, - "server_location": 1, - "server_name": 0 - }, - "renameByName": { - "Time": "", - "server_location": "Location", - "server_name": "Server" - } - } - } - ], - "type": "table" - } - ], + "panels": [], "title": "Averages by Test Server", "type": "row" }, + { + "datasource": "${DS_INFLUXDB-SPEEDTESTS}", + "description": "Shows the avg speeds received by test site", + "fieldConfig": { + "defaults": { + "custom": { + "align": "center", + "displayMode": "color-background", + "filterable": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Mbits" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Ping" + }, + "properties": [ + { + "id": "unit", + "value": "ms" + }, + { + "id": "thresholds", + "value": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 25 + }, + { + "color": "red", + "value": 50 + } + ] + } + }, + { + "id": "custom.displayMode", + "value": "gradient-gauge" + }, + { + "id": "max", + "value": 100 + }, + { + "id": "custom.width" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Test Count" + }, + "properties": [ + { + "id": "unit", + "value": "none" + }, + { + "id": "custom.displayMode", + "value": "gradient-gauge" + }, + { + "id": "max", + "value": 25 + }, + { + "id": "custom.width", + "value": 150 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Location" + }, + "properties": [ + { + "id": "custom.displayMode", + "value": "color-text" + }, + { + "id": "custom.width", + "value": 180 + }, + { + "id": "custom.align", + "value": "left" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Download Avg" + }, + "properties": [ + { + "id": "custom.displayMode", + "value": "gradient-gauge" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 50 + }, + { + "color": "green", + "value": 100 + } + ] + } + }, + { + "id": "max" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Upload Avg" + }, + "properties": [ + { + "id": "custom.displayMode", + "value": "gradient-gauge" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 10 + }, + { + "color": "green", + "value": 50 + } + ] + } + }, + { + "id": "max" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Server" + }, + "properties": [ + { + "id": "custom.width", + "value": 300 + }, + { + "id": "custom.align", + "value": "left" + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 114, + "options": { + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "Ping" + } + ] + }, + "pluginVersion": "7.3.6", + "targets": [ + { + "groupBy": [ + { + "params": [ + "server_name" + ], + "type": "tag" + }, + { + "params": [ + "server_location" + ], + "type": "tag" + }, + { + "params": [ + "server_id" + ], + "type": "tag" + } + ], + "measurement": "download", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT mean(\"bandwidth\") AS \"Download Avg\" FROM \"download\" WHERE $timeFilter GROUP BY \"server_name\", \"server_location\"", + "rawQuery": false, + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "bandwidth" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "Download Avg" + ], + "type": "alias" + } + ], + [ + { + "params": [ + "bandwidth" + ], + "type": "field" + }, + { + "params": [], + "type": "count" + }, + { + "params": [ + "Test Count" + ], + "type": "alias" + } + ] + ], + "tags": [ + { + "key": "namespace", + "operator": "=~", + "value": "/^$namespace$/" + } + ] + }, + { + "groupBy": [ + { + "params": [ + "server_id" + ], + "type": "tag" + } + ], + "measurement": "upload", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT mean(\"bandwidth\") AS \"Upload Avg\" FROM \"upload\" WHERE $timeFilter GROUP BY \"server_id\"", + "rawQuery": false, + "refId": "B", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "bandwidth" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "Upload Avg" + ], + "type": "alias" + } + ] + ], + "tags": [ + { + "key": "namespace", + "operator": "=~", + "value": "/^$namespace$/" + } + ] + }, + { + "groupBy": [ + { + "params": [ + "server_id" + ], + "type": "tag" + } + ], + "measurement": "ping", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT mean(\"bandwidth\") AS \"Upload Avg\" FROM \"upload\" WHERE $timeFilter GROUP BY \"server_id\"", + "rawQuery": false, + "refId": "C", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "latency" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "Ping" + ], + "type": "alias" + } + ] + ], + "tags": [ + { + "key": "namespace", + "operator": "=~", + "value": "/^$namespace$/" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "", + "transformations": [ + { + "id": "seriesToColumns", + "options": { + "byField": "server_id" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Time 1": true, + "Time 2": true, + "Time 3": true, + "server_id": true + }, + "indexByName": { + "Download Avg": 3, + "Ping": 2, + "Test Count": 7, + "Time 1": 6, + "Time 2": 8, + "Time 3": 9, + "Upload Avg": 4, + "server_id": 5, + "server_location": 1, + "server_name": 0 + }, + "renameByName": { + "Time": "", + "server_location": "Location", + "server_name": "Server" + } + } + } + ], + "type": "table" + }, { "collapsed": true, "datasource": null, @@ -2295,7 +2652,7 @@ "h": 1, "w": 24, "x": 0, - "y": 23 + "y": 36 }, "id": 129, "panels": [ @@ -2535,7 +2892,7 @@ "type": "row" } ], - "refresh": "1m", + "refresh": "", "schemaVersion": 26, "style": "dark", "tags": [], @@ -2587,5 +2944,5 @@ "timezone": "", "title": "SpeedTests", "uid": "kLXTiedGz", - "version": 16 + "version": 30 } From 17fb6a22b239ef52cbd4a596f408c0563c78cd8f Mon Sep 17 00:00:00 2001 From: Leigh Phillips Date: Wed, 20 Jan 2021 22:12:32 -0800 Subject: [PATCH 26/27] Update README.md --- README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index d6cbb5b..5ab6832 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ This script will allow you to measure your internet connections speed and consis I built a Grafana dashboard which has been exported into this repo as `GrafanaDash-SpeedTests.json` to import into Grafana for your convenience. -![Grafana Dashboard](https://grafana.com/api/dashboards/13053/images/8976/image) +![GrafanaDashboard](https://user-images.githubusercontent.com/945191/105287048-46f52a80-5b6c-11eb-9e57-038d63b67efb.png) There are some added features to allow some additional details that Ookla provides as tags on your data. Some examples are your current ISP, the interface being used, the server who hosted the test. Overtime, you could identify if some serers are performing better than others. @@ -23,10 +23,9 @@ The variables available are: - INFLUX_DB_DATABASE = default - speedtests - INFLUX_DB_TAGS = default - None * See below for options, '*' widcard for all * - SPEEDTEST_INTERVAL = default - 5 (minutes) -- SPEEDTEST_FAIL_INTERVAL = deafult - 5 (now deprecated) - SPEEDTEST_SERVER_ID = default - {blank} * id from https://c.speedtest.net/speedtest-servers-static.php * - PING_INTERVAL = default - 5 (seconds) -- PING_TARGETS = deafault - 1.1.1.1, 8.8.8.8 (csv of hosts to ping) +- PING_TARGETS = default - 1.1.1.1, 8.8.8.8 (csv of hosts to ping) ### Variable Notes - Intervals are in minutes. *Script will convert it to seconds.* From f8d8ee2407da1fa63d3bfa6d207dbb2a5869438f Mon Sep 17 00:00:00 2001 From: Leigh Phillips Date: Thu, 21 Jan 2021 19:43:07 -0800 Subject: [PATCH 27/27] Update GrafanaDash-SpeedTests.json Data anomaly fixes. --- GrafanaDash-SpeedTests.json | 148 ++++++++++++++++++++++++------------ 1 file changed, 101 insertions(+), 47 deletions(-) diff --git a/GrafanaDash-SpeedTests.json b/GrafanaDash-SpeedTests.json index 901fdd1..cc0749c 100644 --- a/GrafanaDash-SpeedTests.json +++ b/GrafanaDash-SpeedTests.json @@ -59,7 +59,7 @@ "gnetId": 13053, "graphTooltip": 0, "id": null, - "iteration": 1611206261586, + "iteration": 1611279295418, "links": [], "panels": [ { @@ -833,7 +833,7 @@ "y": 7 }, "id": 139, - "interval": "", + "interval": "5s", "options": { "colorMode": "value", "graphMode": "area", @@ -874,7 +874,7 @@ }, { "params": [], - "type": "min" + "type": "mean" } ] ], @@ -963,45 +963,6 @@ "stack": false, "steppedLine": false, "targets": [ - { - "alias": "Latency", - "groupBy": [ - { - "params": [ - "$interval" - ], - "type": "time" - } - ], - "measurement": "pings", - "orderByTime": "ASC", - "policy": "default", - "query": "SELECT mean(\"bytes\") FROM \"download\" WHERE $timeFilter GROUP BY time($__interval) fill(0)", - "rawQuery": false, - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "rtt" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [ - { - "key": "namespace", - "operator": "=~", - "value": "/^$namespace$/" - } - ] - }, { "alias": "Uptime", "groupBy": [ @@ -1029,7 +990,7 @@ }, { "params": [], - "type": "min" + "type": "mean" } ] ], @@ -1041,6 +1002,97 @@ } ], "tz": "" + }, + { + "alias": "Latency", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + } + ], + "measurement": "pings", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT mean(\"rtt\") FROM \"pings\" WHERE (\"namespace\" =~ /^$namespace$/) AND success = 1 AND $timeFilter GROUP BY time($interval)", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "rtt" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "namespace", + "operator": "=~", + "value": "/^$namespace$/" + } + ] + }, + { + "alias": "", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "target" + ], + "type": "tag" + } + ], + "hide": true, + "measurement": "pings", + "orderByTime": "ASC", + "policy": "autogen", + "query": "SELECT mean(\"rtt\") AS \"Latency\" FROM \"pings\" WHERE (\"namespace\" =~ /^$namespace$/) AND success = 1 AND $timeFilter GROUP BY time($interval), \"target\"", + "rawQuery": true, + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "rtt" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "Latency" + ], + "type": "alias" + } + ] + ], + "tags": [ + { + "key": "namespace", + "operator": "=~", + "value": "/^$namespace$/" + } + ] } ], "thresholds": [], @@ -1125,7 +1177,7 @@ "y": 10 }, "id": 140, - "interval": "", + "interval": "5s", "options": { "colorMode": "value", "graphMode": "area", @@ -1154,6 +1206,8 @@ "measurement": "pings", "orderByTime": "ASC", "policy": "default", + "query": "SELECT mean(\"rtt\") FROM \"pings\" WHERE (\"namespace\" =~ /^$namespace$/) AND success = 1 AND $timeFilter GROUP BY time($__interval)", + "rawQuery": true, "refId": "A", "resultFormat": "time_series", "select": [ @@ -2706,7 +2760,7 @@ "h": 7, "w": 24, "x": 0, - "y": 36 + "y": 37 }, "id": 130, "options": { @@ -2892,7 +2946,7 @@ "type": "row" } ], - "refresh": "", + "refresh": false, "schemaVersion": 26, "style": "dark", "tags": [], @@ -2944,5 +2998,5 @@ "timezone": "", "title": "SpeedTests", "uid": "kLXTiedGz", - "version": 30 + "version": 38 }