From 993ab368c61e55745620188f022c2a1dbde145a5 Mon Sep 17 00:00:00 2001 From: Sander Roosendaal Date: Fri, 7 Jul 2023 15:26:01 +0200 Subject: [PATCH] blockers through the db --- rowers/integrations/c2.py | 32 ++++++------ rowers/integrations/nk.py | 31 +++++------ rowers/management/commands/getsyncids.py | 63 +++++++++++++++++++++++ rowers/models.py | 19 ++++++- rowers/tasks.py | 50 +++++++----------- rowers/tests/test_analysis.py | 7 +++ rowers/tests/testdata/testdata.tcx.gz | Bin 4000 -> 3999 bytes rowers/uploads.py | 4 ++ rowers/views/analysisviews.py | 2 + 9 files changed, 142 insertions(+), 66 deletions(-) create mode 100644 rowers/management/commands/getsyncids.py diff --git a/rowers/integrations/c2.py b/rowers/integrations/c2.py index c718a887..14d522f1 100644 --- a/rowers/integrations/c2.py +++ b/rowers/integrations/c2.py @@ -1,5 +1,7 @@ from .integrations import SyncIntegration, NoTokenError -from rowers.models import User, Rower, Workout, TombStone +from rowers.models import User, Rower, Workout, TombStone, SyncRecord +from django.db.utils import IntegrityError + from rowingdata import rowingdata import numpy as np import datetime @@ -358,6 +360,16 @@ class C2Integration(SyncIntegration): def get_workout(self, id, *args, **kwargs): _ = self.open() + + record = SyncRecord( + rower = self.rower, + c2id = id, + ) + try: + record.save() + except IntegrityError: + return 0 + _ = myqueue(queuehigh, handle_c2_getworkout, self.user.id, @@ -413,21 +425,11 @@ class C2Integration(SyncIntegration): workouts = [] c2ids = [item['id'] for item in res.json()['data']] - knownc2ids = uniqify([ - w.uploadedtoc2 for w in Workout.objects.filter(user=self.rower) - ]) - tombstones = [ - t.uploadedtoc2 for t in TombStone.objects.filter(user=self.rower) - ] - parkedids = [] - try: - with open('c2blocked.json', 'r') as c2blocked: - jsondata = json.load(c2blocked) - parkedids = jsondata['ids'] - except: # pragma: no cover - pass - knownc2ids = uniqify(knownc2ids+tombstones+parkedids) + knownc2ids = uniqify([ + record.c2id for record in SyncRecord.objects.filter(rower=r) + ]) + for item in res.json()['data']: d = item['distance'] i = item['id'] diff --git a/rowers/integrations/nk.py b/rowers/integrations/nk.py index 466370b3..10393267 100644 --- a/rowers/integrations/nk.py +++ b/rowers/integrations/nk.py @@ -1,5 +1,6 @@ from .integrations import SyncIntegration, NoTokenError -from rowers.models import User, Rower, Workout, TombStone +from rowers.models import User, Rower, Workout, TombStone, SyncRecord +from django.db.utils import IntegrityError from rowers import mytypes from rowers.nkimportutils import * @@ -88,6 +89,15 @@ class NKIntegration(SyncIntegration): _ = self.open() r = self.rower + record = SyncRecord( + rower = r, + nkid = id, + ) + try: + record.save() + except IntegrityError: + return 0 + before = kwargs.get('before',0) after = kwargs.get('after',0) if not before: @@ -181,25 +191,8 @@ class NKIntegration(SyncIntegration): # get NK IDs nkids = [item['id'] for item in jsondata] knownnkids = uniqify([ - w.uploadedtonk for w in Workout.objects.filter(user=r) + record.nkid for record in SyncRecord.objects.filter(rower=r) ]) - tombstones = [ - t.uploadedtonk for t in TombStone.objects.filter(user=r) - ] - parkedids = [] - try: - with open('nkblocked.json', 'r') as nkblocked: - try: - jsondatal = json.load(nkblocked) - except: - jsondatal = { - 'ids':[] - } - parkedids = jsondatal['ids'] - except FileNotFoundError: # pragma: no cover - pass - - knownnkids = uniqify(knownnkids+tombstones+parkedids) workouts = [] diff --git a/rowers/management/commands/getsyncids.py b/rowers/management/commands/getsyncids.py new file mode 100644 index 00000000..422f1f23 --- /dev/null +++ b/rowers/management/commands/getsyncids.py @@ -0,0 +1,63 @@ +from django.core.management.base import BaseCommand +from rowers.models import Workout, TombStone, SyncRecord +from django.utils import timezone +from django.db.utils import IntegrityError + +class Command(BaseCommand): + def handle(self, *args, **options): + ws = Workout.objects.all() + aantal = ws.count() + counter = 0 + print('----- Workouts ---------') + for w in ws: + record = SyncRecord( + workout = w, + ) + if w.uploadedtostrava: + record.stravaid = w.uploadedtostrava + if w.uploadedtotp: + record.tpid = w.uploadedtotp + if w.uploadedtonk: + record.nkid = w.uploadedtonk + if w.uploadedtosporttracks: + record.sporttracksid = w.uploadedtosporttracks + if w.uploadedtoc2: + record.c2id = w.uploadedtoc2 + + try: + record.save() + except IntegrityError: + pass + + counter += 1 + if counter % 10 == 0: + print(counter,'/',aantal) + + print('----- Tombstones -------') + ts = TombStone.objects.all() + aantal = ts.count() + counter = 0 + for w in ts: + record = SyncRecord( + ) + if w.uploadedtostrava: + record.stravaid = w.uploadedtostrava + if w.uploadedtotp: + record.tpid = w.uploadedtotp + if w.uploadedtonk: + record.nkid = w.uploadedtonk + if w.uploadedtosporttracks: + record.sporttracksid = w.uploadedtosporttracks + if w.uploadedtoc2: + record.c2id = w.uploadedtoc2 + + try: + record.save() + except IntegrityError: + pass + + counter += 1 + if counter % 10 == 0: + print(counter,'/',aantal) + + diff --git a/rowers/models.py b/rowers/models.py index 770acd9e..98a9c73b 100644 --- a/rowers/models.py +++ b/rowers/models.py @@ -3447,6 +3447,7 @@ rpechoices = ( (10, '10 Max Effort (You can barely remember your name, you would rather rip out your toenails than go through this)') ) + class Workout(models.Model): workouttypes = mytypes.workouttypes workoutsources = mytypes.workoutsources @@ -3608,7 +3609,6 @@ class TombStone(models.Model): uploadedtotp = models.BigIntegerField(default=0) uploadedtonk = models.BigIntegerField(default=0) - @receiver(models.signals.pre_delete, sender=Workout) def create_tombstone_on_delete(sender, instance, **kwargs): t = TombStone( @@ -3623,6 +3623,23 @@ def create_tombstone_on_delete(sender, instance, **kwargs): # delete files belonging to workout instance # related GraphImage objects should be deleted automatically +class SyncRecord(models.Model): + workout = models.ForeignKey(Workout, on_delete=models.CASCADE, null=True) + rower = models.ForeignKey(Rower, on_delete=models.CASCADE, null=True) + stravaid = models.BigIntegerField(unique=True,null=True,default=None) + sporttracksid = models.BigIntegerField(unique=True,null=True,default=None) + nkid = models.BigIntegerField(unique=True,null=True,default=None) + c2id = models.BigIntegerField(unique=True,null=True,default=None) + tpid = models.BigIntegerField(unique=True,null=True,default=None) + + def save(self, *args, **kwargs): + if self.workout: + self.rower = self.workout.user + return super(SyncRecord, self).save(*args, **kwargs) + + + + @receiver(models.signals.post_delete, sender=Workout) def auto_delete_file_on_delete(sender, instance, **kwargs): diff --git a/rowers/tasks.py b/rowers/tasks.py index 857e844c..6a78d09f 100644 --- a/rowers/tasks.py +++ b/rowers/tasks.py @@ -15,7 +15,7 @@ application = get_wsgi_application() from rowers.models import ( Workout, GeoPolygon, GeoPoint, GeoCourse, VirtualRaceResult, CourseTestResult, Rower, - GraphImage, + GraphImage, SyncRecord ) import math @@ -84,6 +84,7 @@ import rowers.rowing_workout_metrics_pb2 as metrics_pb2 import rowers.rowing_workout_metrics_pb2_grpc as metrics_pb2_grpc from django.conf import settings +from django.db.utils import IntegrityError # extra read of config @@ -522,6 +523,9 @@ def handle_c2_sync(workoutid, url, headers, data, debug=False, **kwargs): workout.uploadedtoc2 = c2id workout.save() + record = SyncRecord(workout=workout,c2id=c2id) + record.save() + return 1 def splitstdata(lijst): @@ -3402,24 +3406,20 @@ def handle_nk_async_workout(alldata, userid, nktoken, nkid, delaysec, defaulttim # dologging('nklog.log','NK Workout ID {id}'.format(id=workoutid)) workout = Workout.objects.get(id=workoutid) newnkid = workout.uploadedtonk + sr = SyncRecord.objects.filter(nkid=newnkid) + if len(sr): + sr[0].workout = workout + try: + sr[0].save() + except IntegrityError: + pass + else: + sr = SyncRecord(workout=workout,nkid=newnkid) + try: + sr.save() + except IntegrityError: + pass - parkedids = [] - try: - with open('nkblocked.json', 'r') as nkblocked: - jsondata = json.load(nkblocked) - parkedids = jsondata['ids'] - except FileNotFoundError: # pragma: no cover - pass - - newparkedids = [id for id in parkedids if id != newnkid] - with open('nkblocked.json', 'wt') as nkblocked: - tdata = {'ids': newparkedids} - nkblocked.seek(0) - json.dump(tdata, nkblocked) - - # evt update workout summary - - # return return workoutid @@ -3693,20 +3693,8 @@ def handle_c2_async_workout(alldata, userid, c2token, c2id, delaysec, workout = Workout.objects.get(id=workoutid) newc2id = workout.uploadedtoc2 - parkedids = [] - with open('c2blocked.json', 'a+') as c2blocked: - try: - jsondata = json.load(c2blocked) - parkedids = jsondata['ids'] - except JSONDecodeError: # pragma: no cover - parkedids = [] - + record = SyncRecord(workout=workout,c2id=newc2id) - newparkedids = [id for id in parkedids if id != newc2id] - with open('c2blocked.json', 'wt') as c2blocked: - tdata = {'ids': newparkedids} - c2blocked.seek(0) - json.dump(tdata, c2blocked) # set distance, time workout = Workout.objects.get(id=workoutid) diff --git a/rowers/tests/test_analysis.py b/rowers/tests/test_analysis.py index 8835ac82..c97eaf45 100644 --- a/rowers/tests/test_analysis.py +++ b/rowers/tests/test_analysis.py @@ -743,6 +743,13 @@ class WorkoutStatsTestNew(TestCase): self.c = Client() self.user_workouts = WorkoutFactory.create_batch(5, user=self.r) + + today = datetime.date.today() + i = 0 + for w in self.user_workouts: + w.date = today-datetime.timedelta(days=i) + i += 1 + w.save() self.factory = RequestFactory() self.password = faker.word() self.u.set_password(self.password) diff --git a/rowers/tests/testdata/testdata.tcx.gz b/rowers/tests/testdata/testdata.tcx.gz index 27485f4657a9f615c609be58395df8f5b3b4bf3a..19b96a23a8d4b89676613dd7f7591167510d2c06 100644 GIT binary patch delta 3913 zcmV-P54P~2AD(%=B;d0$CFCKM$_eZx`?xmgAoBMZP_5Jz&V)6Fv zTXWo9oUYd9=4K?&hVDa*~RgK0C4g9(-kMrx2L$bT5tMpeY8wRZC3lc^FBF|@D~BtlL!GZe?LFF z=ad-D+`ug$9LjrX7K<*K|K;E~wPtHU~9N}*O`v3=vwA07^pLFMY&-?D8 zf3Z4U-oN`-$6j729s1&|?@sbz&zDDM>zA8DVSljrYU`&19<6q7(9>o5v72Fd+~(+!Bb<_500#Feif zueYyi|Ne0MRxA_Q%Pm(QI^1HyI2KvWP{Vdf1V$7Mcm0%#63Yoa6!SHlHOF1d$>7ws!QIjRK(pGVpNUt zX^7y0+~aJxJ6jR=nB?7x*Km&|<(*+R+cdn!v$21@Az8>xZ#s{<-dG}&ue#V@M0`!&(Ddx4Hc13dO8!T zlCL0^jM7<}(P)nqk&h-Ic&kdgYvgA@p2@g~QAIu{=N2SCe&bJ@ zy=6escyiKy5_eZg}fMG2r8KuDjKnAk3yaeTNKqiAGoN7=bOxrLf#pSC@Ptcf3~Riw#ocBYG9Q{%Hwt-gD5BGNZ!No``SziW z_G6G|*tW#d==RpSqPl%(@%d56lW`1D)dWw2j)2BgCRs!=LCsmYixYe?Z+UGU?LK0ytfXp<=#$T=8=sAB2ARm;dIDz z^W+SYcRA}?)d|rN7S-+NM82lcJ{mv_n)tk9Dq2PDTYP>D@;Uc5V9@lvb(V_e-p+}9 ze?|5@Swjw4HO~u8Cij+G{rn(#Z_pyCMtes^?fDszcT^Gi+&O_*)8F2CE;`T8iF`%6 zebUxAqH5JL7q~@fXFy&IfT+5hyT~m{JN?m~y%EuMLiEtc-;{hsy1fO%UR0hOkG7RHLWtBAZe2m~6dE?}69%s2AGpF03GJiwz zu_E$fASg$BTXdzuS$T3Q8tu^=FQV#}5?s;MN$2Ff^;8deg4`R6HJ+SMc8yA*Mt&6X zY>;$4FheN1YT2B~*EG+Qm>?K6e;Jq|V$sYDn$bM(D1h-J%d=7gRrLLak+A9Ye+cGbd$ejN0<6RKlXW)`Lyt7-T%e?TvJz*mKA5rv9I z>*u8ARb@F(U$hVItuiv98Fz!|tpOm`Nqw-B zL66P28$|CDdR-HZoXu4fJva2DppPb6rAvpv6*catdGp3VpOp28y0ROj?D%g|KY8?G zoY$hKvaQnQB=t}cfBIlVNYwxvDOrTRp`W%($LG$N)`_T$J4y$M9&7Du4@)j zEV^*%tOk6l2)#34Ntcn!z>`_yrp@)k=!1!(tiEBiMO`{=f36<|J)wz2pi0nUn$DVL z&4)jX-kQM5>KkS^nWkwiOJ^9pH{J@XG<>#Y*SVoqb2kS1oM9=btFLD%+fId=nl}o1 zHUb!RQt!fK27GJ@T7&4Fv8Y?i5ZT9~jnk;v^`oGVh9X*>h9Am?Z@78$#z1c=_s`OK z^Ek$)=1qUVf45>ll!aFoVKVXh+LEy=3fkJFu_48%LQmA@`st$=Fj2X1FA@`cM)2kUMDUf4XYeST=maZNMJ`J)&V2r;>Vs zvdh);jMO}+2)*65R#2&VQn-!Moa*asMd-aDs|?x-Ls4&@Z3F%o=m`uGBdgF`n9SOS z&6_ujep_Tss(5{}2wIW#45Z%Wt{PE?-ceEGu65NI2Ytv=?{x#d=gCNY8}J9wGnyc* zi`PrAe?>L?7PK7)y)_OIRb4eC+UCuhz9W~t@k~|~v~}Pnld93wyg~HIPO}bmSvn3~ z8}O%|o0rhD5z0;)4SJeR2}Wz9G>qOE3pvqJgPubh@Tc#{wK1V5QWah~8z)m~+p=^9 z(F+;Y`tY5DHrLMpy%-{`n&+WqYW1vUM z4ONs<&t*$6papG5K`*(X3hAWYO3}z2G&OG&^ue%T&`G_xqNy59%^L+hCF72itrfg; zf6b9Qb(c=eEfYbzrmgpWGIKq(pzR>~Tx89l3g7tPn_WKx^wCJN?sQTg{bcNVYHHpP zdL|>*>cTexLbK~<1e|`k?&Um&_<pvuw-p=hP;tB9#r=LYwPHLGN?*e~l77 z`^l_s+2;B|^m!#~MpAk6qV+|qZ>aUn8wEXY7co~EcTr+n-E!jIyo}!E!Yd_u7mG@L zYm_<;`j`u^bas6V(3+=B+?$usZ&$KbM($$7)){*S=&eaJy=nmz0%$#QZ;IZJh5p0V zj~fmaPrCEV56{l}<>v6alm3UFe@~XD%XPo^{dUv$dtJW=@BE*9~UjulrYL7gt`B2j}M}tD~+@KmGE>xIFh)CrjV`^m=vj^77~Yf0o|Ibm*0b z^I^-=?my|0*;#P)^Vhqr)@iesPy1~*Tt3<_&VTSC-SpwF%adidStbto(}zcQ<-QJ{ zbnD~Cj}Cu+mf39o+-?6Qy~WR7pZ>8-cLs0|VE6yQ;>s7hC%=76hj+H9SL^f~9wwB_ Xi%02c{E;qwc;|n0fBm_{1AqYln?U!j delta 3914 zcmV-Q54G^0AD|xxABzYGg|MfQ2OkqEHRn>NE{<_TAge_JHVP=4w^fljBY&UXou2Hy zSzc^bXX|@+(ZJokpAPRFJiq8x>(%=B!E)U%FCKP%_oCY@_tMVm&Aq#?`u==>v3UFT ztvT*4PFHJlbauK(Uv7?GEl<16;`+lEZ|Lss6#Lz~Z#c{Sqki>f)xZDcUB6uC8@zIY zcbnB`XL!rz?BaMq0J!+$$ra;g+f)3oT5tMpeY8wRZC3lc^FBV2@D~BllL!GZe>*?B z=ad-D+`ufq!LjrU^fc!x49C_d3J~;oJu(oP@uf83q#J?p!R z{`u;3dGGFD9ea79bm;T5zB|c>JzE}~tzT{qh5fB#^LuyldU=R8U1ay?rz{@{Pd6a$nj5(05?8){ zwBEj|{rki1Td_=JFSlHMkS~|%0Ur|P$GH3Z3dM`n@o~DfW&i5z<>8ZVfBm{UNfbZZ zdUfya=)a%c{dUXw*=pTi-JHKj@4z>_i`(5kKk|!LLW~`LO|}QzlE768%q55%&ZU!370(N_ta4?&0R#sV;f9QW1A&h*34l zry+t1a*wm&?rcTeW0H3(Uc)_>ly`>NaQ78)x3nGSor1d!h6-}WTX4s!xCf|+yCm*O zs5IKVH*O-_>yr1t6>;|_cyJnh-no)y9MgQb`+B$w7$4AT`1DXP!UN8S-CQtTueW}TZBp)7g61$nBAJXJHr(>^3q zPIRUmqLlRAu$_<0vs6UhnwXS&mCTcvP>{ULhFG?U zO1^?vGD>G@Mx#AeL_V5;;H@g{u92Ssc_!l`Miu#-oLi9me~kS0HI4Qxh8=j#JWsYP zc{?NW6-8_~7y$61AumueI&W{|x94Wn4UtdsYaOc6ut;J_AKt9USLDO9Y#7mMqO|0T zX4p6KBawFotde|y$;fzdzw6!Kz#A*f_tsA$BdkspV=Hz7$pYREfoGJiwzTn~8!BLRY{ zNrva5GVhyiKMHv^Y*AG6eBh!Qo^LWg3VCNRqNrp(f7+tn+a~klkl*Iss#>7gvK5uL zB9G07H$*6-6-U}p@>f7y|wI$=G%uh z+K)k=VcQZ*quX2Sit6^E#pg#MPsTAsRTDfZOMXrjm98T4VyrN$=6P4NmUdR;r6Tgd zB<`vVe@t7pqEevA{21gdnxt_$QnfrS+Ur(kR7EYeBJym!(?w}*j3sk&=0v`p8XL~J zx6UJKMteb+jLbKCewchT6d(o_`Pg*(=|^mYa<>#!lvWs;Za;mQ7i&V&?Nu^w%Vu52 zR!chyd2fVu5gU=RsnkRe}7Z*wj%Omg4b17iO0#r=bOwAlNVzN zRn=7z%GTAj9+=~h4~7s`of9OMt+DY$f=dpjrc ze-+vDWDPlF)jTgWncQ1$_49+|y+Mnl8tokwwdZF<-cd#5bLRwNO@Djmx#&DUC-N2P z_DNggh^ke~T;LX^odJ0<0HW%0?jpA+?es@`_C`e43DH9%e^c@m>Gl>3dr^6EJlc{P z-mUWRnua$_o^$t8mG{;sOE9(!%*m4%f8(t~O{~r*-JoEsPH;x{d`0nj8;nIpO-_za zZ;PD>`PzJVp&s&rxt3P7riGWH_49L*`IAKYYg`{unJATRrX#w&ezJ{Dbtd{#0q6_K~bf~aIZe?ZZ- zsAnbft|Ic@AP{J*x`1IaGT+D#laI!w!Co`k2MR^)`8kdDp&s%`##)S;tn0wB$@~q; z$BM{{fuJ1iZPAqqXXVMMXtYOfyojn>N^nJ2C!Le`)>A#?336{N)_8J4*)=MK8u?Mk zvq94Jzzm`2s%3K`U(-BKVuE1Qe`H{Wh($9qXh!qAuZVmwl4M>}X%T^%%uhX5C;N=X zGO02%B61@?^@vR(Zw;k?D)Llx9h+Q#+A=dnAn$V?o~j>Sd(a~klmBJ`Z*dIHs4j}RtdJ1Avg5x={p8V$ zabAm_%C<_Ilhi{+f9Qh|Ayorxq+}8LhJM;E9iKa6S|_41?kFh;GLiN<*p{UqL~jjy zUHhE8Fves$CpEi%7`->v8j1upvNS@%7QzjfRpKni!Cj45Iq}ox~^GB zvFO63vl{TJBJ|FHC0#}?15aj+n>N=EqYoyEvigS67Io>gf4P1X^n@l7fhs|ZX*z3~ zH6Q*kdTRnJt8bXyWSXY6ES+KW-gqml((u`qUFU{c&D|L2bB3j$uD+h7Y&#WdYThX5 z*$80NNxch`8St?sXbqxw#-eU5Lu4O|Hcq2v*N=ie8j5Ij8h$7nzTxK08w0(i+&@d_ z&EpuGnm7Fcf8UA$Q5IfVgvrF~YfHwiC}?Yw#)cH53O!Mq>!*)iz(nQ3#b}v|M(&yq ze;o8|qC?W5w>+8U&6*>382z?oqF1STR%~bm6`}WrtTJdT3`M9P}Yez1I!+o+l&qZNMKy&uD_M zE?zIee-_p7ThMkK^wu~;RCU#mXqz{0`i@-o#xq$}(AI&QOsYmx^9Io;JIy-OW$8F{ zZNQ&;ZeBvqMkqUJH0WtMB^a%Z(lB~wEaXH>4SEi3z@NS&*T#gNNL6^{Y@AG`ZOhUb zL@#7m>%(^r+FU;a^kRs#YMzIdkvsLmtB_kJe`4CELmzN54ON<&H-J9omWhBEb#{HE zmXF$$)Ll9;w@d`>nzr8i$;|cCg0_R`bCETJDtzOEZ+86*&_^T5y3Y0> z>?gCjWt-~<(dU({8A;{Mi`EydzM{hac^Eizg@{%8M%uQTW9PUptmN?^r{6=2%z=Iy(xM>7Wxlc zKW;c!JnqgfKRi9_mz%@yPWm5ze>qv6F4z6u_uEb1?{)niyz_th5CH#tfehar_dgyi zb_d+qo$2Fm%%3kmt#;oZJh(_4&n`af`?O1M&&Ao>O}Bp8ot$i+R{FSFAHVGSE}ggA zy