From: Sam Moore Date: Mon, 19 Aug 2013 04:27:52 +0000 (+0800) Subject: Add data performance tests, revise block diagram, add pthread_mutex_init X-Git-Url: https://git.ucc.asn.au/?a=commitdiff_plain;h=cdc9404739cec84843bc96031e957041d19608a6;hp=cb4eb97e6d35fcb1e1493682afb2fe4a4bda29b6;p=matches%2FMCTX3420.git Add data performance tests, revise block diagram, add pthread_mutex_init --- diff --git a/reports/week3/block_diagram.png b/reports/week3/block_diagram.png new file mode 100644 index 0000000..379a485 Binary files /dev/null and b/reports/week3/block_diagram.png differ diff --git a/reports/week3/block_diagram.svg b/reports/week3/block_diagram.svg new file mode 100644 index 0000000..8bbe47a --- /dev/null +++ b/reports/week3/block_diagram.svg @@ -0,0 +1,698 @@ + + + + + + + + + + image/svg+xml + + + + + + + "High level" Server(Beaglebone / RPi) + + + Read Analogue InputWrite Digital Output + + LEGENDHardware (Implementation)Software (Implementation)Existing software can be used + + + Image Processing(OpenCV) + + + + Video Cameras + + + + + Networking(Wireless Chip, Wired connection) + OS provided TCP/IP + + + + Remote Client(PC or laptop controlled by user) + Client side scripting (JavaScript) + web browser + + + + Server control API (FastCGI) + webserver installed in OS (nginx) + + + + + Actuators + + + ADC, DAC, Digital I/O + + + + + OS provided drivers (if we're lucky) + + + Existing OS (GNU/Linux) + + + USB + + + + + + + + + + + + Electronics, Amplification + + + + + Sensors + + Pneumatics + + + + + + + + + Multithreaded server with watchdog (pthreads for C) + + Data collection & transfer + (sensors: binary/CSV/sqlite)(actuators: single value, PID control) + + + + + Data Storage(USB Harddrive) + + + + + diff --git a/server/sensor.c b/server/sensor.c index b746134..83ffaaa 100644 --- a/server/sensor.c +++ b/server/sensor.c @@ -70,6 +70,8 @@ void Sensor_Init(Sensor * s, int id) { Fatal("Too many sensors! FILENAMESIZE is %d; increase it and recompile.", FILENAMESIZE); } + + pthread_mutex_init(&(s->mutex), NULL); sprintf(filename, "%d", s->id); unlink(filename); //TODO: Move old files somewhere diff --git a/testing/data_performance/Makefile b/testing/data_performance/Makefile new file mode 100644 index 0000000..d938c99 --- /dev/null +++ b/testing/data_performance/Makefile @@ -0,0 +1,27 @@ +# Makefile for rpi side server +CXX = gcc +FLAGS = -std=c99 -Wall -Werror -pedantic -g +LIB = -lpthread -lsqlite3 +BIN = binfile csv sqlite +RM = rm -f + + + +all : $(BIN) + +% : %.c + $(CXX) $(FLAGS) -o $@ $< $(LIB) + + + +clean : + $(RM) $(BIN) + $(RM) *.o + +clean_full: #cleans up all backup files + $(RM) $(BIN) $(OBJ) $(LINKOBJ) + $(RM) *.*~ + $(RM) *~ + + + diff --git a/testing/data_performance/README b/testing/data_performance/README new file mode 100644 index 0000000..ac48a8f --- /dev/null +++ b/testing/data_performance/README @@ -0,0 +1,3 @@ +A few tests of performance for different methods of storing/transferring sensor data. +The important thing to remember is that the data is stored and accessed sequentially. +A database, whilst a useful thing and definitely capable of being used for this, is probably overkill. diff --git a/testing/data_performance/binfile.c b/testing/data_performance/binfile.c new file mode 100644 index 0000000..aae8ea2 --- /dev/null +++ b/testing/data_performance/binfile.c @@ -0,0 +1,56 @@ +#include +#include +#include +#include + +typedef struct +{ + float time; + float value; +} DataPoint; + + + + +int main(int argc, char ** argv) +{ + assert(argc == 3); + + int bufsiz = atoi(argv[1]); + int numpoints = atoi(argv[2]); + assert(bufsiz > 0); + DataPoint * buffer = (DataPoint*)(calloc(bufsiz, sizeof(DataPoint))); + + + + FILE * file = fopen("data.bin", "wb"); + + struct timeval start_time; + gettimeofday(&start_time, NULL); + + int i = 0; + while (i < numpoints) + { + int j = 0; + for (j = 0; j < bufsiz && i < numpoints; ++j) + { + buffer[j].time = i; + buffer[j].value = i; + + } + i += j; + assert(fwrite(buffer, sizeof(DataPoint), j, file) == j); + + + } + + struct timeval end_time; + gettimeofday(&end_time, NULL); + + + fclose(file); + free(buffer); + float time_elapsed = (float)(end_time.tv_sec - start_time.tv_sec) + 1e-6*(end_time.tv_usec - start_time.tv_usec); + printf("%f\n", time_elapsed); + return 0; +} diff --git a/testing/data_performance/csv.c b/testing/data_performance/csv.c new file mode 100644 index 0000000..3a8155f --- /dev/null +++ b/testing/data_performance/csv.c @@ -0,0 +1,61 @@ +#include +#include +#include +#include + +typedef struct +{ + float time; + float value; +} DataPoint; + + + + +int main(int argc, char ** argv) +{ + assert(argc == 3); + + int bufsiz = atoi(argv[1]); + int numpoints = atoi(argv[2]); + assert(bufsiz > 0); + DataPoint * buffer = (DataPoint*)(calloc(bufsiz, sizeof(DataPoint))); + + + FILE * file = fopen("data.csv", "w"); + + struct timeval start_time; + gettimeofday(&start_time, NULL); + + + int i = 0; + while (i < numpoints) + { + int j = 0; + for (j = 0; j < bufsiz && i < numpoints; ++j) + { + buffer[j].time = i; + buffer[j].value = i; + + } + i += j; + + for (int k = 0; k < j; ++k) + { + fprintf(file, "%f,%f\n", buffer[i].time, buffer[i].value); + } + } + + + + struct timeval end_time; + gettimeofday(&end_time, NULL); + + + fclose(file); + + free(buffer); + float time_elapsed = (float)(end_time.tv_sec - start_time.tv_sec) + 1e-6*(end_time.tv_usec - start_time.tv_usec); + printf("%f\n", time_elapsed); + return 0; +} diff --git a/testing/data_performance/main.c b/testing/data_performance/main.c new file mode 100644 index 0000000..4485875 --- /dev/null +++ b/testing/data_performance/main.c @@ -0,0 +1,55 @@ +#include +#include +#include +#include + +typedef struct +{ + float time; + float value; +} DataPoint; + + + + +int main(int argc, char ** argv) +{ + assert(argc == 3); + + int bufsiz = atoi(argv[1]); + int numpoints = atoi(argv[2]); + assert(bufsiz > 0); + DataPoint * buffer = (DataPoint*)(calloc(bufsiz, sizeof(DataPoint))); + + struct timeval start_time; + gettimeofday(&start_time, NULL); + + FILE * file = fopen("data.bin", "wb"); + + int i = 0; + while (i < numpoints) + { + int j = 0; + for (j = 0; j < bufsiz && i < numpoints; ++j) + { + buffer[j].time = i; + buffer[j].value = i; + + } + i += j; + assert(fwrite(buffer, sizeof(DataPoint), j, file) == j); + + + } + + fclose(file); + + + struct timeval end_time; + gettimeofday(&end_time, NULL); + + free(buffer); + float time_elapsed = (float)(end_time.tv_sec - start_time.tv_sec) + 1e-6*(end_time.tv_usec - start_time.tv_usec); + printf("%f\n", time_elapsed); + return 0; +} diff --git a/testing/data_performance/plot.py b/testing/data_performance/plot.py new file mode 100755 index 0000000..90e4d6b --- /dev/null +++ b/testing/data_performance/plot.py @@ -0,0 +1,44 @@ +#!/usr/bin/python + +import sys +import os +import Gnuplot +import subprocess +import numpy + +methods = ["sqlite", "csv", "binfile"] +data = {} +averages = 100 +g = Gnuplot.Gnuplot() + + +for program in methods: + data.update({program : []}) + + numpoints = 1 + while (numpoints < 10000): + bufsiz = 1 + while (bufsiz < 2): + + run = ["./"+program, str(bufsiz), str(numpoints)] + times = [] + for i in xrange(averages): + p = subprocess.Popen(run, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + times.append(float(p.stdout.readline().strip(" \r\n"))) + + times = numpy.array(times) + data[program].append([numpoints,bufsiz,numpy.mean(times), numpy.std(times)]) + + bufsiz *= 10 + numpoints += 100 + + g("set xlabel \"Data Points\"") + g("set ylabel \"Total Time (s)\"") + g("set title \"Time to Store Data Points\"") + g.replot(Gnuplot.Data(data[program], title=program, with_="lp",using="1:3")) + +print "Done!" +sys.stdin.readline() + + + diff --git a/testing/data_performance/sqlite.c b/testing/data_performance/sqlite.c new file mode 100644 index 0000000..b0ffb36 --- /dev/null +++ b/testing/data_performance/sqlite.c @@ -0,0 +1,65 @@ +#include +#include +#include +#include +#include + +typedef struct +{ + float time; + float value; +} DataPoint; + + + + +int main(int argc, char ** argv) +{ + assert(argc == 3); + + int bufsiz = atoi(argv[1]); + int numpoints = atoi(argv[2]); + assert(bufsiz > 0); + DataPoint * buffer = (DataPoint*)(calloc(bufsiz, sizeof(DataPoint))); + + + + sqlite3 * db; + sqlite3_open("sqlite.db", &db); + + + char query[BUFSIZ]; + char * query_value = query+sprintf(query,"insert into sensor values("); + + + struct timeval start_time; + gettimeofday(&start_time, NULL); + + int i = 0; + while (i < numpoints) + { + int j = 0; + for (j = 0; j < bufsiz && i < numpoints; ++j) + { + buffer[j].time = i; + buffer[j].value = i; + + } + i += j; + for (int k = 0; k < j; ++k) + { + sprintf(query_value,"%f,%f)", buffer[k].time, buffer[k].value); + sqlite3_exec(db, query, NULL, NULL, NULL); + } + } + + struct timeval end_time; + gettimeofday(&end_time, NULL); + + + sqlite3_close(db); + free(buffer); + float time_elapsed = (float)(end_time.tv_sec - start_time.tv_sec) + 1e-6*(end_time.tv_usec - start_time.tv_usec); + printf("%f\n", time_elapsed); + return 0; +} diff --git a/testing/data_performance/storespeed.png b/testing/data_performance/storespeed.png new file mode 100644 index 0000000..3f50e42 Binary files /dev/null and b/testing/data_performance/storespeed.png differ