from datetime import datetime, timezone, timedelta
import random
def genTimestampToNs(f, n, m):
file = open(f, "w+")
for i in range(n):
rawYear = random.randint(1700, 2026)
rawMonth = random.randint(1, 12)
rawDay = random.randint(1, 28) # Max 28 days a month for simplicity
rawH = random.randint(0, 23)
rawM = random.randint(0, 59)
rawS = random.randint(0, 59)
rawUs = 0
for j in range(m):
rawUs = rawUs * 10 + random.randint(0, 9)
dt = datetime(rawYear, rawMonth, rawDay, rawH, rawM, rawS, rawUs, tzinfo=timezone.utc)
epoch = datetime(1970, 1, 1, tzinfo=timezone.utc)
delta = dt - epoch
file.write(f'{rawYear:04d}-{rawMonth:02d}-{rawDay:02d}T{rawH:02d}:{rawM:02d}:{rawS:02d}.{rawUs:06d}Z' + ' ' + str((delta.days * 86400 + delta.seconds) * 1_000_000_000 + delta.microseconds * 1000) + '\n')
file.close()
def genPx(f, n, m):
file = open(f, "w+")
for i in range(n):
rawPx = random.randint(0, 200000)
rawFrac = ''
for j in range(random.randint(0, m)):
rawFrac = rawFrac + str(random.randint(0, 9))
trailZeros = 0
for j in range(len(rawFrac)):
if rawFrac[len(rawFrac) - 1 - j] == '0':
trailZeros += 1
else:
break
rawFrac = rawFrac[:len(rawFrac) - trailZeros]
px = f'{rawPx}.{rawFrac}' if rawFrac else str(rawPx)
pad = ''
for i in range(m - len(rawFrac)):
pad += '0'
scaled = rawPx * (10 ** m) + int((rawFrac + pad) or 0)
file.write(f'{px} {scaled}\n')
file.close()
def genSlidingWindow(f, sz, cap, n, lo, hi, m, k, pxMag):
file = open(f, "w+")
fileMem = []
basePrice = 85000
maxPrice = 94000
minPrice = 76000
magPriceBoundMin = 5
magPriceBoundMax = 50
priceLowCdInit = int(n * 0.05) # 5% of max interations will be set as min price decrease CD
priceHighCdInit = int(n * 0.05) # same as above
priceLowCd = 0
priceHighCd = 0
rawPrice = basePrice
rawYear = 2026
rawMonth = 10
rawDay = 25
rawH = 10
rawM = 43
rawS = 11
rawUs = 492348
for i in range(n):
dt = datetime(rawYear, rawMonth, rawDay, rawH, rawM, rawS, rawUs, tzinfo=timezone.utc)
epoch = datetime(1970, 1, 1, tzinfo=timezone.utc)
delta = dt - epoch
rawNs = (delta.days * 86400 + delta.seconds) * 1_000_000_000 + delta.microseconds * 1000
us = random.randint(lo, hi)
rawNs = rawNs + us
sec = rawNs // 1_000_000_000
nano = rawNs % 1_000_000_000
dt = datetime(1970, 1, 1, tzinfo=timezone.utc) + timedelta(seconds=sec)
rawYear = dt.year
rawMonth = dt.month
rawDay = dt.day
rawH = dt.hour
rawM = dt.minute
rawS = dt.second
rawUs = dt.microsecond
rawUs = nano // 1000
ts = f'{rawYear:04d}-{rawMonth:02d}-{rawDay:02d}T{rawH:02d}:{rawM:02d}:{rawS:02d}.{rawUs:06d}Z'
ns = str(rawNs)
dir = random.randint(0, 1)
if rawPrice <= minPrice:
dir = 1
if rawPrice >= maxPrice:
dir = 0
magPx = int(random.randint(magPriceBoundMin, magPriceBoundMax) * 0.2) * 5 # divide by 5 then mul by 5, simple trick to clamp price increase/decrease to a mul of 5
rawPrice = rawPrice + (-1 if dir == 0 else 1) * magPx
rawFrac = ''
for j in range(m):
rawFrac = rawFrac + str(random.randint(0, 9))
trailZeros = 0
for j in range(len(rawFrac)):
if rawFrac[len(rawFrac) - 1 - j] == '0':
trailZeros += 1
else:
break
rawFrac = rawFrac[:len(rawFrac) - trailZeros]
px = f'{rawPrice}.{rawFrac}' if rawFrac else str(rawPrice)
pad = ''
for i in range(pxMag - len(rawFrac)):
pad += '0'
scaled = rawPrice * (10 ** pxMag) + int((rawFrac + pad) or 0)
tradeId = ''
for j in range(k):
tradeId = tradeId + str(random.randint(0, 9))
real = ts + ' ' + tradeId + ' ' + px
expected = ns + ' ' + str(scaled)
fileMem.append(real + ' ' + expected)
#Rule: N = not on window, Y = on window, H = head of window, T = tail of window
windowSize = 0
totalSize = 0
tail = fileMem[-1]
fileMem[-1] += " T"
head = None
tailNs = tail.split()[3]
for i in range(len(fileMem) - 1):
mem = fileMem[i]
memArr = mem.split()
timeNs = memArr[3]
totalSize += 1
if int(tailNs) - int(timeNs) > sz:
fileMem[i] += " N"
continue
if head == None:
fileMem[i] += " H"
head = mem
else:
fileMem[i] += " Y"
windowSize += 1
# Window size by nano time - window capacity - window max capacity - window elements count -
maxCap = cap
while windowSize >= maxCap: maxCap *= 2
file.write(str(sz) + ' ' + str(cap) + ' ' + str(windowSize + 1) + ' ' + str(maxCap) + ' ' + str(totalSize + 1) + '\n')
for i in range(len(fileMem)):
file.write(fileMem[i] + ('\n' if i != len(fileMem) - 1 else ''))
file.close()
genTimestampToNs("test_data/HelperTest-ts.testdata", 1000000, 6)
genPx("test_data/HelperTest-px.testdata", 1000000, 6)
genSlidingWindow("test_data/SlidingWindow-short.testdata", 60_000_000_000, 1024, 10000, 200000, 500000, 2, 10, 6)
genSlidingWindow("test_data/SlidingWindow-long.testdata", 3_600_000_000_000, 1024, 1000000, 1_000_000, 20_000_000, 2, 10, 6)