diff --git a/Base/CCXT-PlaceOrder.future b/Base/CCXT-PlaceOrder.future index b7269cd..5f295c6 100755 --- a/Base/CCXT-PlaceOrder.future +++ b/Base/CCXT-PlaceOrder.future @@ -87,6 +87,13 @@ def main(): relay.JRLog.Write('PlaceOrder FUTURE '+relay.Version) + # Check for OneShot situation + + if ("Conditional" in relay.Active or "Conditional" in relay.Order) \ + and ("ConditionalOneShot" in relay.Active or "ConditionalOneShot" in relay.Order): + if relay.OliverTwistOneShot(relay.Order)==True: + relay.JRLog.Error("Conditional OneShot", f"{relay.Order['Exchange']}/{relay.Order['Account']}/{relay.Order['Asset']} already being managed by OliverTwist ") + # Now lets get down to business. The processed order is in: # relay.Order diff --git a/Base/CCXT-PlaceOrder.margin b/Base/CCXT-PlaceOrder.margin index 939beb2..0a5134a 100755 --- a/Base/CCXT-PlaceOrder.margin +++ b/Base/CCXT-PlaceOrder.margin @@ -87,6 +87,13 @@ def main(): relay.JRLog.Write('PlaceOrder MARGIN '+relay.Version) + # Check for OneShot situation + + if ("Conditional" in relay.Active or "Conditional" in relay.Order) \ + and ("ConditionalOneShot" in relay.Active or "ConditionalOneShot" in relay.Order): + if relay.OliverTwistOneShot(relay.Order)==True: + relay.JRLog.Error("Conditional OneShot", f"{relay.Order['Exchange']}/{relay.Order['Account']}/{relay.Order['Asset']} already being managed by OliverTwist ") + # Now lets get down to business. The processed order is in: # relay.Order diff --git a/Base/CCXT-PlaceOrder.spot b/Base/CCXT-PlaceOrder.spot index b7a23a8..a0f3cfc 100755 --- a/Base/CCXT-PlaceOrder.spot +++ b/Base/CCXT-PlaceOrder.spot @@ -93,6 +93,13 @@ def main(): if ("Conditional" in relay.Active or "Conditional" in relay.Order) and relay.Order['Action']!='buy': relay.JRLog.Error("Conditional", f"{relay.Order['Action']}: ignored") + # Check for OneShot situation + + if ("Conditional" in relay.Active or "Conditional" in relay.Order) \ + and ("ConditionalOneShot" in relay.Active or "ConditionalOneShot" in relay.Order): + if relay.OliverTwistOneShot(relay.Order)==True: + relay.JRLog.Error("Conditional OneShot", f"{relay.Order['Exchange']}/{relay.Order['Account']}/{relay.Order['Asset']} already being managed by OliverTwist ") + # Now lets get down to business. The processed order is in: # relay.Order diff --git a/Base/CCXT-PlaceOrder.swap b/Base/CCXT-PlaceOrder.swap index ab2c13d..be06739 100755 --- a/Base/CCXT-PlaceOrder.swap +++ b/Base/CCXT-PlaceOrder.swap @@ -87,6 +87,13 @@ def main(): relay.JRLog.Write('PlaceOrder SWAP '+relay.Version) + # Check for OneShot situation + + if ("Conditional" in relay.Active or "Conditional" in relay.Order) \ + and ("ConditionalOneShot" in relay.Active or "ConditionalOneShot" in relay.Order): + if relay.OliverTwistOneShot(relay.Order)==True: + relay.JRLog.Error("Conditional OneShot", f"{relay.Order['Exchange']}/{relay.Order['Account']}/{relay.Order['Asset']} already being managed by OliverTwist ") + # Now lets get down to business. The processed order is in: # relay.Order diff --git a/Base/Conditional.ccxt b/Base/Conditional.ccxt index 4576278..1db35a1 100755 --- a/Base/Conditional.ccxt +++ b/Base/Conditional.ccxt @@ -22,6 +22,35 @@ import time import JRRsupport import JackrabbitRelay as JRR +# Timeout before Locker auto-deletes this order result + +OliverTwistTimeout=(15*60) + +# Write the result to Locker memory so parent knows we are finished. + +def FinishOrphan(Key,lID,mID,State): + # Get the lock read and set up the memory key. Locker doesn't care and which class this order belongs to. OliverTwist will + # match the ID to the right class list, orphan or conditional. + + OliverTwistLock=JRRsupport.Locker("OliverTwist",ID=lID) + Memory=JRRsupport.Locker(Key,ID=mID) + + OliverTwistLock.Lock() + + State=State.lower() + + if State!='delete': + # Return this order to a waiting state + Memory.Put(OliverTwistTimeout*100,"Waiting") + elif State=='delete': + # This order has been processed and needs to be removed from the system. + Memory.Put(OliverTwistTimeout*100,"Delete") + + OliverTwistLock.Unlock() + + # We're done. This child has completed its task + sys.exit(0) + # Get the order ID. If there isn't an ID, the order FAILED. def GetOrderID(res): @@ -78,7 +107,7 @@ def main(): data=sys.stdin.read().strip() Orphan=json.loads(data) - # Use Relay to process and validate the order + # Use Relay to process and validate the order, must be a string if type(Orphan['Order']) is dict: order=json.dumps(Orphan['Order']) else: @@ -92,6 +121,8 @@ def main(): # Handle OANDa's weird order id sequencing id=Orphan['ID'] saction=relay.Order['SellAction'].lower() + if type(Orphan['Response']) is str: + Orphan['Response']=json.loads(Orphan['Response']) oDetail=Orphan['Response']['Details'] # Manage average and close extire position @@ -102,26 +133,6 @@ def main(): price=float(oDetail['price']) amount=float(oDetail['amount']) - # Check to see if we have enough balance, if not then delete this order. Deal with futures as well. - - base=relay.Markets[relay.Order['Asset']]['base'].upper() - if relay.Order['Market']=='spot': - bal=relay.GetBalance(Base=base) - else: - bal=relay.GetPositions(symbols=[relay.Order['Asset']]) - if amount>bal: - relay.JRLog.Write(f"{id}: Amount {amount:.8f} > Balance {bal:.8f} {base}, purge",stdOut=False) - rData={} - rData['Key']=Orphan['Key'] - rData['lID']=Orphan['lID'] - rData['mID']=Orphan['mID'] - rData['Class']=Orphan['Class'] - rData['Status']='Delete' - print(json.dumps(rData)) - # Flush and exit - sys.stdout.flush() - sys.exit(0) - # Process the position # We need to check TakeProfit and StopLoss. If one of them is hit, we need to build and order and backfeed it in @@ -132,6 +143,63 @@ def main(): # Get Ticker ticker=relay.GetTicker(symbol=relay.Order['Asset']) + # Check to see if we have enough balance, if not then delete this order. Deal with futures as well. + + base=relay.Markets[relay.Order['Asset']]['base'].upper() + if relay.Order['Market']=='spot': + bal=relay.GetBalance(Base=base) + else: + bal=relay.GetPositions(symbols=[relay.Order['Asset']]) + + # if ticker is below price and spread, purge and re-purchase + + if amount>bal: + if "ConditionalRepurchase" in relay.Active: + relay.JRLog.Write(f"Original: {json.dumps(relay.Order)}",stdOut=False) + + # figure out direction and build replacement order + makeNewOrder=False + if dir=='long': + makeNewOrder=ticker['Bid']<(price-ticker['Spread']) + else: + makeNewOrder=ticker['Ask']>(price+ticker['Spread']) + + if makeNewOrder==True: + # Copy original order + newOrder=relay.Order + newOrder['OliverTwist']=f'Repurchase @/<{price}' + """ + newOrder['Exchange']=relay.Order['Exchange'] + newOrder['Account']=relay.Order['Account'] + newOrder['Market']=relay.Order['Market'] + newOrder['Asset']=relay.Order['Asset'] + newOrder['Action']=relay.Order['SellAction'] + newOrder['Price']=str(strikePrice) + newOrder['Base']=str(amount) + """ + if 'OrderType' in relay.Order: + newOrder['OrderType']=relay.Order['OrderType'] + else: + newOrder['OrderType']='market' + relay.JRLog.Write(f"New: {json.dumps(newOrder)}",stdOut=False) + newOrder['Identity']=relay.Identity['Identity'] + + # Feed the new order to Relay + result=relay.SendWebhook(newOrder) + oid=GetOrderID(result) + if oid!=None: + # Repurchase was successful, remove this order + FinishOrphan(Orphan['Key'],Orphan['lID'],Orphan['mID'],'Delete') + else: + # Something went wrong, leave it and try again later + FinishOrphan(Orphan['Key'],Orphan['lID'],Orphan['mID'],Orphan['Status']) + else: + # Not time to make a new purchase, leave it + FinishOrphan(Orphan['Key'],Orphan['lID'],Orphan['mID'],'Delete') + else: + relay.JRLog.Write(f"{id}: Amount {amount:.8f} > Balance {bal:.8f} {base}, purge",stdOut=False) + FinishOrphan(Orphan['Key'],Orphan['lID'],Orphan['mID'],'Delete') + # Fsilsafe, in the WORST way possible. Do NOT leave a take profit out of the order. At this stage, the whole thing is # an absolute nightmare to fix. The is a very brutal way of dealing with poor user choices. if 'TakeProfit' not in relay.Order: @@ -194,7 +262,7 @@ def main(): # relay.JRLog.Write(f"{id}: {json.dumps(newOrder)}",stdOut=False) - newOrder['Identity']=relay.Identity['Identity'] + newOrder['Identity']=relay.Active['Identity'] # Feed the new order to Relay result=relay.SendWebhook(newOrder) @@ -203,53 +271,21 @@ def main(): resp=relay.GetOrderDetails(id=oid,symbol=relay.Order['Asset']) # Order must be closed as it succedded newOrder['ID']=oid - rData={} - rData['Key']=Orphan['Key'] - rData['lID']=Orphan['lID'] - rData['mID']=Orphan['mID'] - rData['Class']=Orphan['Class'] - rData['Status']='Delete' - print(json.dumps(rData)) relay.WriteLedger(Order=newOrder,Response=resp) + FinishOrphan(Orphan['Key'],Orphan['lID'],Orphan['mID'],'Delete') else: # Give OliverTwist a response relay.JRLog.Write(f"{id}: Order failed",stdOut=False) - rData={} - rData['Key']=Orphan['Key'] - rData['lID']=Orphan['lID'] - rData['mID']=Orphan['mID'] - rData['Class']=Orphan['Class'] - rData['Status']=Orphan['Status'] - print(json.dumps(rData)) - # Flush and exit - sys.stdout.flush() - sys.exit(0) + FinishOrphan(Orphan['Key'],Orphan['lID'],Orphan['mID'],Orphan['Status']) else: # Strike did not happen - rData={} - rData['Key']=Orphan['Key'] - rData['lID']=Orphan['lID'] - rData['mID']=Orphan['mID'] - rData['Class']=Orphan['Class'] - rData['Status']=Orphan['Status'] - print(json.dumps(rData)) - #relay.JRLog.Write(f"{id}: {cur['state']}",stdOut=False) - # Flush and exit - sys.stdout.flush() - sys.exit(0) + FinishOrphan(Orphan['Key'],Orphan['lID'],Orphan['mID'],Orphan['Status']) except Exception as e: # Something went wrong - rData={} - rData['Key']=Orphan['Key'] - rData['lID']=Orphan['lID'] - rData['mID']=Orphan['mID'] - rData['Class']=Orphan['Class'] - rData['Status']=Orphan['Status'] - print(json.dumps(rData)) relay.JRLog.Write(f"{Orphan['Key']}: Code Error - {str(e)}",stdOut=False) - # Flush and exit - sys.stdout.flush() - sys.exit(0) + if 'Diagnostics' in relay.Active: + relay.JRLog.Write(f"{Orphan['Key']}: {data}",stdOut=False) + FinishOrphan(Orphan['Key'],Orphan['lID'],Orphan['mID'],Orphan['Status']) if __name__ == '__main__': main() diff --git a/Base/Conditional.oanda b/Base/Conditional.oanda index 27a2345..e88e48b 100755 --- a/Base/Conditional.oanda +++ b/Base/Conditional.oanda @@ -22,6 +22,35 @@ import time import JRRsupport import JackrabbitRelay as JRR +# Timeout before Locker auto-deletes this order result + +OliverTwistTimeout=(15*60) + +# Write the result to Locker memory so parent knows we are finished. + +def FinishOrphan(Key,lID,mID,State): + # Get the lock read and set up the memory key. Locker doesn't care and which class this order belongs to. OliverTwist will + # match the ID to the right class list, orphan or conditional. + + OliverTwistLock=JRRsupport.Locker("OliverTwist",ID=lID) + Memory=JRRsupport.Locker(Key,ID=mID) + + OliverTwistLock.Lock() + + State=State.lower() + + if State!='delete': + # Return this order to a waiting state + Memory.Put(OliverTwistTimeout*100,"Waiting") + elif State=='delete': + # This order has been processed and needs to be removed from the system. + Memory.Put(OliverTwistTimeout*100,"Delete") + + OliverTwistLock.Unlock() + + # We're done. This child has completed its task + sys.exit(0) + # Get the order ID. If there isn't an ID, the order FAILED. def GetOrderID(res): @@ -200,7 +229,7 @@ def main(): # relay.JRLog.Write(f"{id}: {json.dumps(newOrder)}",stdOut=False) - newOrder['Identity']=relay.Identity['Identity'] + newOrder['Identity']=relay.Active['Identity'] # Feed the new order to Relay result=relay.SendWebhook(newOrder) @@ -209,66 +238,22 @@ def main(): resp=relay.GetOrderDetails(OrderID=oid) # Order must be closed as it succedded newOrder['ID']=oid - rData={} - rData['Key']=Orphan['Key'] - rData['lID']=Orphan['lID'] - rData['mID']=Orphan['mID'] - rData['Class']=Orphan['Class'] - rData['Status']='Delete' - print(json.dumps(rData)) relay.WriteLedger(Order=newOrder,Response=resp) + FinishOrphan(Orphan['Key'],Orphan['lID'],Orphan['mID'],'Delete') else: # Give OliverTwist a response relay.JRLog.Write(f"{id}: Order failed",stdOut=False) - rData={} - rData['Key']=Orphan['Key'] - rData['lID']=Orphan['lID'] - rData['mID']=Orphan['mID'] - rData['Class']=Orphan['Class'] - rData['Status']=Orphan['Status'] - print(json.dumps(rData)) - #relay.JRLog.Write(f"{id}: {cur['state']}",stdOut=False) - # Flush and exit - sys.stdout.flush() - sys.exit(0) + FinishOrphan(Orphan['Key'],Orphan['lID'],Orphan['mID'],cur['state']) else: # Strike did not happen - rData={} - rData['Key']=Orphan['Key'] - rData['lID']=Orphan['lID'] - rData['mID']=Orphan['mID'] - rData['Class']=Orphan['Class'] - rData['Status']=Orphan['Status'] - print(json.dumps(rData)) - #relay.JRLog.Write(f"{id}: {cur['state']}",stdOut=False) - # Flush and exit - sys.stdout.flush() - sys.exit(0) + FinishOrphan(Orphan['Key'],Orphan['lID'],Orphan['mID'],cur['state']) # Fall through. No order matching the ID. - rData={} - rData['Key']=Orphan['Key'] - rData['lID']=Orphan['lID'] - rData['mID']=Orphan['mID'] - rData['Class']=Orphan['Class'] - rData['Status']="Delete" - print(json.dumps(rData)) - sys.stdout.flush() - sys.exit(0) - #relay.JRLog.Write(f"{id}: {cur['state']}",stdOut=False) + FinishOrphan(Orphan['Key'],Orphan['lID'],Orphan['mID'],'Delete') except Exception as e: # Something broke or went horrible wrong - rData={} - rData['Key']=Orphan['Key'] - rData['lID']=Orphan['lID'] - rData['mID']=Orphan['mID'] - rData['Class']=Orphan['Class'] - rData['Status']=Orphan['Status'] - print(json.dumps(rData)) relay.JRLog.Write(f"{Orphan['Key']}: Code Error - {str(e)}",stdOut=False) - # Flush and exit - sys.stdout.flush() - sys.exit(0) + FinishOrphan(Orphan['Key'],Orphan['lID'],Orphan['mID'],cur['state']) if __name__ == '__main__': main() diff --git a/Base/JackrabbitLocker b/Base/JackrabbitLocker index 8c88b14..98bd078 100755 --- a/Base/JackrabbitLocker +++ b/Base/JackrabbitLocker @@ -28,7 +28,7 @@ import json import JRRsupport -Version="0.0.0.1.340" +Version="0.0.0.1.370" BaseDirectory='/home/JackrabbitRelay2/Base' ConfigDirectory='/home/JackrabbitRelay2/Config' LogDirectory="/home/JackrabbitRelay2/Logs" diff --git a/Base/JackrabbitOliverTwist b/Base/JackrabbitOliverTwist index 2ad0127..c957f34 100755 --- a/Base/JackrabbitOliverTwist +++ b/Base/JackrabbitOliverTwist @@ -33,13 +33,12 @@ import os import time import json import random -import multiprocessing import subprocess import JRRsupport import JackrabbitRelay as JRR -Version="0.0.0.1.340" +Version="0.0.0.1.370" BaseDirectory='/home/JackrabbitRelay2/Base' DataDirectory='/home/JackrabbitRelay2/Data' ConfigDirectory='/home/JackrabbitRelay2/Config' @@ -51,24 +50,23 @@ OrphanReceiver=DataDirectory+'/OliverTwist.Orphan.Receiver' ConditionalStorehouse=DataDirectory+'/OliverTwist.Conditional.Storehouse' OrphanStorehouse=DataDirectory+'/OliverTwist.Orphans.Storehouse' -# This keeps track of all sub-processes for cleanup - -PoolResults=[] - # Set up the logging system JRLog=JRR.JackrabbitLog() # Set up signal interceptor -interceptor=JRRsupport.SignalInterceptor(Log=JRLog) +if __name__=='__main__': + interceptor=JRRsupport.SignalInterceptor(Log=JRLog,IsMain=True) +else: + interceptor=JRRsupport.SignalInterceptor(Log=JRLog,IsMain=False) # The timeout setting for the memory locks. If it takes longer then 15 minutes to # check the status of an order, there is a major problem with the exchange/broker. # Testing has show that only a few seconds is actually requred. The extended amount # is just to compensate for an overloaded server. -OliverTwistTimeout=(1*60) +OliverTwistTimeout=(15*60) # This lock guards the lists. Both PlaceOrder.olivertwist and # JackrabbitOliverTwist will access this file. Collisions must not be allowed to @@ -129,112 +127,6 @@ def GetID(): pw+=oc return pw -# Receive an update on an orphan and update, delete, or add to the main list. This is NOT part of the -# main process, but rather a child process. NONE o the global or main variables are available. The ONLY -# information know is what is EXPLICTLY given. - -def OrphanUpdate(result): - global JRLog - - if result!=None: - # result in JSON format - # Locking required to protect memory structures - - # At this stage, rData should NEVER be None or undefined. If it is, then - # memory was thrashed. - - try: - rData=json.loads(result) - except: - #JRLog.Write(f"OrphanUpdate ERROR: |{result}|",stdOut=False) - return - - OliverTwistLock=JRRsupport.Locker("OliverTwist",ID=rData['lID']) - if rData['Class']=='Orphan': - OrphanMemory=JRRsupport.Locker(rData['Key'],ID=rData['mID']) - else: - ConditionalMemory=JRRsupport.Locker(rData['Key'],ID=rData['mID']) - OliverTwistLock.Lock() - - # if returned status is open, replace the response, if it has - # changed or the orphan record doesn't have a response - - rStatus=rData['Status'].lower() - if rStatus!='delete': # open/pnding - # Nothing changed, go back to waiting - if rData['Class']=='Orphan': - OrphanMemory.Put(OliverTwistTimeout,"Waiting") - else: - ConditionalMemory.Put(OliverTwistTimeout,"Waiting") - # if the status returnd is delete, flag for parent - # do NOT delete here, will trash orphan list. - elif rStatus=='delete': # or closed - if rData['Class']=='Orphan': - OrphanMemory.Put(OliverTwistTimeout,"Delete") - else: - ConditionalMemory.Put(OliverTwistTimeout,"Delete") - else: - JRLog.Write(f"ERROR: {rStatus}/rdata['Class']/{rData['Key']}",stdOut=False) - OliverTwistLock.Unlock() - -# This spins up the orphan manager to ensure each orphan is handled. This is NOT part of the main process, but rather a child -# process. NONE o the global or main variables are available. The ONLY information know is what is EXPLICTLY given. - -# An Orphan transactor will process orphans. A relatively easy process compared to the Conditional transactor. The Condiional -# transactor will need to back feed orders into Relay, usingsimilat techniques compared to the PlaceOrder system. The -# Conditional framework will be a logistical nightmare. Emphasis has to be on speed in processing the conditions. - -def ProcessOrphan(Orphan): - global JRLog - - # Figure out which transactor is needed - if Orphan['Class']=='Orphan': - OrphanManager=BaseDirectory+'/Orphan.'+Orphan['Framework'] - else: - OrphanManager=BaseDirectory+'/Conditional.'+Orphan['Framework'] - - # Diagnostics only - # fn=f"/tmp/{Orphan['Framework']},{Orphan['Key']}.OliverTwist" - # if not os.path.exists(fn): - # JRRsupport.WriteFile(fn,json.dumps(Orphan)+'\n') - - if os.path.exists(OrphanManager): - transactor=[ OrphanManager ] - - # Time to get down to business - - try: - payload=json.dumps(Orphan).encode() -# if Orphan['Class']=='Conditional': -# JRRsupport.WriteFile('x9',json.dumps(Orphan)) -# sys.exit(3) -# JRLog.Write(f"{payload}",stdOut=False) - - subp=subprocess.Popen(transactor,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE) - subp.stdin.write(payload) - res=subp.communicate()[0].decode().strip() - - # response will be proper json - return res - except: - # This should never happen, but if it dowes, go back to waiting state - if rdata['Class']=='Orphan': - OrphanMemory=JRRsupport.Locker(Orphan['Key'],ID=Orphan['mID']) - OrphanMemory.Put(OliverTwistTimeout,"Waiting") - else: - ConditionalMemory=JRRsupport.Locker(Orphan['Key'],ID=Orphan['mID']) - ConditionalMemory.Put(OliverTwistTimeout,"Waiting") - return None - - # This should never happen, but if it dowes, go back to waiting state - if rdata['Class']=='Orphan': - OrphanMemory=JRRsupport.Locker(Orphan['Key'],ID=Orphan['mID']) - OrphanMemory.Put(OliverTwistTimeout,"Waiting") - else: - ConditionalMemory=JRRsupport.Locker(Orphan['Key'],ID=Orphan['mID']) - ConditionalMemory.Put(OliverTwistTimeout,"Waiting") - return None - # Read the complete list stored on disk, if it exists. Supports both orphans and conditionals. def ReadOrphanList(cl=None): @@ -291,7 +183,7 @@ def ReadOrphanList(cl=None): OrphanList.insert(json.dumps(Orphan)) OrphanMemory[Orphan['Key']]=JRRsupport.Locker(Orphan['Key']) - OrphanMemory[Orphan['Key']].Put(OliverTwistTimeout,"Waiting") + OrphanMemory[Orphan['Key']].Put(OliverTwistTimeout*100,"Waiting") OrphanListChanged=True else: @@ -299,7 +191,7 @@ def ReadOrphanList(cl=None): ConditionalList.insert(json.dumps(Orphan)) ConditionalMemory[Orphan['Key']]=JRRsupport.Locker(Orphan['Key']) - ConditionalMemory[Orphan['Key']].Put(OliverTwistTimeout,"Waiting") + ConditionalMemory[Orphan['Key']].Put(OliverTwistTimeout*100,"Waiting") ConditionalListChanged=True @@ -380,7 +272,7 @@ def ReceiveOrphans(cl): try: Orphan=json.loads(Entry) except: -# JRLog.Write(f"Bad orphan: {Entry}") + JRLog.Write(f"Broken: {Entry}") continue if 'Order' in Orphan: @@ -400,29 +292,83 @@ def ReceiveOrphans(cl): OrphanList.insert(json.dumps(Orphan)) OrphanMemory[Orphan['Key']]=JRRsupport.Locker(Orphan['Key']) - OrphanMemory[Orphan['Key']].Put(OliverTwistTimeout,"Waiting") + OrphanMemory[Orphan['Key']].Put(OliverTwistTimeout*100,"Waiting") OrphanListChanged=True else: Orphan['Class']='Conditional' ConditionalList.insert(json.dumps(Orphan)) ConditionalMemory[Orphan['Key']]=JRRsupport.Locker(Orphan['Key']) - ConditionalMemory[Orphan['Key']].Put(OliverTwistTimeout,"Waiting") + ConditionalMemory[Orphan['Key']].Put(OliverTwistTimeout*100,"Waiting") ConditionalListChanged=True rc+=1 - JRLog.Write(f"{rc} {cl.lower()}n(s) received") + if rc>0: + JRLog.Write(f"{rc} {cl.lower()}n(s) received") OliverTwistLock.Unlock() +# This spins up the orphan manager to ensure each orphan is handled. This is NOT part of the main process, but rather a child +# process. NONE o the global or main variables are available. The ONLY information know is what is EXPLICTLY given. + +# An Orphan transactor will process orphans. A relatively easy process compared to the Conditional transactor. The Condiional +# transactor will need to back feed orders into Relay, usingsimilat techniques compared to the PlaceOrder system. The +# Conditional framework will be a logistical nightmare. Emphasis has to be on speed in processing the conditions. + +def ProcessOrphan(**kwargs): + global JRLog + + # The entire kwargs payload is the orphan JSON... + Orphan=kwargs + + # Figure out which transactor is needed + if Orphan['Class']=='Orphan': + OrphanManager=BaseDirectory+'/Orphan.'+Orphan['Framework'] + else: + OrphanManager=BaseDirectory+'/Conditional.'+Orphan['Framework'] + + # Diagnostics only + # fn=f"/tmp/{Orphan['Framework']},{Orphan['Key']}.OliverTwist" + # if not os.path.exists(fn): + # JRRsupport.WriteFile(fn,json.dumps(Orphan)+'\n') + + if os.path.exists(OrphanManager): + transactor=[ OrphanManager ] + + # Time to get down to business + + try: + payload=json.dumps(Orphan).encode() +# if Orphan['Class']=='Conditional': +# JRRsupport.WriteFile('x9',json.dumps(Orphan)) +# sys.exit(3) +# JRLog.Write(f"{payload}",stdOut=False) + + subp=subprocess.Popen(transactor,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE) + subp.stdin.write(payload) + subp.communicate() # [0].decode().strip() + + # response will be in Jackrabbit Locker as a memory element + return None + except Exception as e: + # This should never happen, but if it dowes, go back to waiting state. Class doesn't matter here + Memory=JRRsupport.Locker(Orphan['Key'],ID=Orphan['mID']) + Memory.Put(OliverTwistTimeout*100,"Waiting") + return None + + # This should never happen, but if it dowes, go back to waiting state. Class doesn't matter here + Memory=JRRsupport.Locker(Orphan['Key'],ID=Orphan['mID']) + Memory.Put(OliverTwistTimeout*100,"Waiting") + return None + # Process each child. Handles both orphans and conditionals # # memData will be eithewr OrphanMemory or ConfitionalMemory -def ProcessChild(ProcessPool,CurList,cur,memData,CurListChanged): - global PoolResults - +def ProcessChild(CurList,cur,memData,CurListChanged): data=json.loads(cur.GetData()) + # Get result from Locker + OliverTwistLock.Lock() sData=json.loads(memData[data['Key']].Get()) OliverTwistLock.Unlock() @@ -477,18 +423,18 @@ def ProcessChild(ProcessPool,CurList,cur,memData,CurListChanged): if status=='waiting': data['lID']=OliverTwistLock.ID data['mID']=memData[data['Key']].ID - memData[data['Key']].Put(OliverTwistTimeout,"Running") - PoolResults.append(ProcessPool.apply_async(ProcessOrphan,args=(data,),callback=OrphanUpdate)) - - # Process any signals received during critical section - interceptor.SafeExit() - # Clean up dead processes - multiprocessing.active_children() - + memData[data['Key']].Put(OliverTwistTimeout*100,"Running") +# PoolResults.append(ProcessPool.apply_async(ProcessOrphan,args=(data,),callback=OrphanUpdate)) + interceptor.StartProcess(ProcessOrphan,kwargs=data) cur=cur.GetNext() OliverTwistLock.Unlock() - return ProcessPool,CurList,cur,CurListChanged + # Process any signals received during critical section + interceptor.SafeExit() + + # Return the updated list information + + return CurList,cur,CurListChanged ### ### Main driver @@ -506,15 +452,11 @@ def main(): JRLog.Write('OliverTwist '+Version) - NumberProcesses=multiprocessing.cpu_count() + NumberProcesses=os.cpu_count() if len(sys.argv)>1: NumberProcesses=int(sys.argv[1]) JRLog.Write(f'Spawning {NumberProcesses} sub-processes') - # Establish the process pool to handle the orphans - multiprocessing.set_start_method("spawn") - ProcessPool=multiprocessing.Pool(processes=NumberProcesses,maxtasksperchild=1) - # Load saved orphans into memory ReadOrphanList('Orphan') ReadOrphanList('Conditional') @@ -528,7 +470,7 @@ def main(): ReceiveOrphans('Conditional') cor=ConditionalList.GetHead() while cor!=None: - ProcessPool,ConditionalList,cor,ConditionalListChanged=ProcessChild(ProcessPool,ConditionalList,cor,ConditionalMemory,ConditionalListChanged) + ConditionalList,cor,ConditionalListChanged=ProcessChild(ConditionalList,cor,ConditionalMemory,ConditionalListChanged) # Update Conditional storehouse, if needed if ConditionalListChanged==True: @@ -537,6 +479,11 @@ def main(): ConditionalListChanged=False OliverTwistLock.Unlock() + # Only allow "NumberProcesses" children to run as once. + + while interceptor.GetChildren()>(NumberProcesses-1): + JRRsupport.ElasticSleep(1) + # Process any signals received during critical section interceptor.SafeExit() @@ -547,7 +494,7 @@ def main(): if cur==None: cur=OrphanList.GetHead() - ProcessPool,OrphanList,cur,OrphanListChanged=ProcessChild(ProcessPool,OrphanList,cur,OrphanMemory,OrphanListChanged) + OrphanList,cur,OrphanListChanged=ProcessChild(OrphanList,cur,OrphanMemory,OrphanListChanged) # Save the orphan list to disk, ONLY IF it actually changed @@ -557,15 +504,13 @@ def main(): OrphanListChanged=False OliverTwistLock.Unlock() - # Process any signals received during critical section - interceptor.SafeExit() + # Only allow "NumberProcesses" children to run as once. - # every cycle, try to clean up any completed processes. + while interceptor.GetChildren()>(NumberProcesses-1): + JRRsupport.ElasticSleep(1) - for res in list(PoolResults): - if not res.ready(): - res.get(timeout=None) - PoolResults.remove(res) + # Process any signals received during critical section + interceptor.SafeExit() # Brief rest to maintain control over server JRRsupport.ElasticSleep(1) diff --git a/Base/JackrabbitRelay b/Base/JackrabbitRelay index 3b37ec7..93fee88 100755 --- a/Base/JackrabbitRelay +++ b/Base/JackrabbitRelay @@ -16,7 +16,7 @@ import json import JRRsupport -Version="0.0.0.1.340" +Version="0.0.0.1.370" BaseDirectory='/home/JackrabbitRelay2/Base' ConfigDirectory='/home/JackrabbitRelay2/Config' LogDirectory="/home/JackrabbitRelay2/Logs" diff --git a/Base/Library/JRRccxt.py b/Base/Library/JRRccxt.py index 8e52820..b0fd2d1 100755 --- a/Base/Library/JRRccxt.py +++ b/Base/Library/JRRccxt.py @@ -677,7 +677,7 @@ def MakeOrphanOrder(self,id,Order): Orphan['Order']=json.dumps(Order) orphanLock.Lock() - JRRsupport.AppendFile(OrphanReceiver,json.dumps(Orphan)) + JRRsupport.AppendFile(OrphanReceiver,json.dumps(Orphan)+'\n') orphanLock.Unlock() # Create a conditional order and deliver to OliverTwist @@ -686,7 +686,10 @@ def MakeConditionalOrder(self,id,Order): ConditionalReceiver=self.DataDirectory+'/OliverTwist.Conditional.Receiver' orphanLock=JRRsupport.Locker("OliverTwist") - resp=Order['Response'] + if type(Order['Response'])==dict: + resp=json.dumps(Order['Response']) + else: + resp=Order['Response'] Order.pop('Response',None) # Strip Identity @@ -702,7 +705,7 @@ def MakeConditionalOrder(self,id,Order): Conditional['Response']=resp orphanLock.Lock() - JRRsupport.AppendFile(ConditionalReceiver,json.dumps(Conditional)) + JRRsupport.AppendFile(ConditionalReceiver,json.dumps(Conditional)+'\n') orphanLock.Unlock() # Make ledger entry. Record everything for accounting purposes diff --git a/Base/Library/JRRoanda.py b/Base/Library/JRRoanda.py index 5f53a6e..53c2187 100644 --- a/Base/Library/JRRoanda.py +++ b/Base/Library/JRRoanda.py @@ -470,6 +470,8 @@ def GetOrderDetails(self,**kwargs): return None + # Make an orphan order + def MakeOrphanOrder(self,id,Order): OrphanReceiver=self.DataDirectory+'/OliverTwist.Orphan.Receiver' orphanLock=JRRsupport.Locker("OliverTwist") @@ -486,7 +488,7 @@ def MakeOrphanOrder(self,id,Order): Orphan['Order']=json.dumps(Order) orphanLock.Lock() - JRRsupport.AppendFile(OrphanReceiver,json.dumps(Orphan)) + JRRsupport.AppendFile(OrphanReceiver,json.dumps(Orphan)+'\n') orphanLock.Unlock() # Create a conditional order and deliver to OliverTwist @@ -511,7 +513,7 @@ def MakeConditionalOrder(self,id,Order): Conditional['Response']=resp orphanLock.Lock() - JRRsupport.AppendFile(ConditionalReceiver,json.dumps(Conditional)) + JRRsupport.AppendFile(ConditionalReceiver,json.dumps(Conditional)+'\n') orphanLock.Unlock() # Make ledger entry with every detail. diff --git a/Base/Library/JRRsupport.py b/Base/Library/JRRsupport.py index 669a19c..576bbed 100755 --- a/Base/Library/JRRsupport.py +++ b/Base/Library/JRRsupport.py @@ -7,16 +7,23 @@ import sys sys.path.append('/home/JackrabbitRelay2/Base/Library') -from multiprocessing import current_process, active_children import os import signal +import psutil import datetime import time import random import socket import json -import psutil + +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import sys +import os import signal +import time +import random # Signal Interceptor for critical areas @@ -25,39 +32,52 @@ # and things go south real quick. Extra care MUST BE and IS taken to stay away from INIT. class SignalInterceptor(): - def __init__(self,Log=None): - # Signals not to trap + def __init__(self,Log=None,IsMain=True): + # Signals not to trap, list(signal.Signals) noTrap=[signal.SIGCHLD,signal.SIGCONT,signal.SIGTSTP,signal.SIGWINCH] - # list(signal.Signals) + self.critical=False + self.IsParent=False + self.IsChild=False self.original={} self.triggered={} self.Log=Log - if current_process().name=='MainProcess': + if IsMain==True: + self.IsParent=True self.parent_id=os.getpid() else: + self.IsChild=True self.parent_id=os.getppid() # Stay away from INIT if self.parent_id==1: self.parent_id=os.getpid() # Set all signals to myself. + for sig in signal.valid_signals(): self.triggered[sig]=False try: self.original[sig]=signal.getsignal(sig) - if current_process().name=='MainProcess': + if IsMain==True: if sig not in noTrap: signal.signal(sig,self.ProcessSignal) else: # Reset child process signal.signal(sig,signal.SIG_IGN) except: pass + # Prent gets task of killing dead child processes - if current_process().name=='MainProcess': - signal.signal(signal.SIGALRM,self.mpCleaner) - signal.alarm(3) + + if IsMain==True: + signal.signal(signal.SIGCHLD,self.SignalChild) + + # For use with Jackrabbit Relay + + def SetLog(self,Log=None): + self.Log=Log + + # If a logging has ben set, use it. Otherwise, just print to screen. def ShowSignalMessage(self,lm): if self.Log!=None: @@ -65,71 +85,47 @@ def ShowSignalMessage(self,lm): else: print(lm) + # Exit the program. If parent, exit child processes. If Child, signal siblings and parent + def SignalInterrupt(self,signal_num): mypid=os.getpid() self.ShowSignalMessage(f'Parent: {self.parent_id} self: {mypid}') - # Only parent takes to children parent = psutil.Process(self.parent_id) - if self.parent_id==mypid and len(parent.children())>1: - if len(parent.children(recursive=True))>1: - for child in parent.children(recursive=True): - if child.pid!=mypid: - self.ShowSignalMessage(f'Exiting child: {child.pid}') - # send signal to children - os.kill(child.pid,9) - active_children() + nChildren=len(parent.children(recursive=False)) + + # Only parent takes to children + if self.parent_id==mypid and nChildren>0: + for child in parent.children(recursive=False): + if child.pid!=mypid: + self.ShowSignalMessage(f'Signaling child: {child.pid}') + # send signal to children + os.kill(child.pid,2) # Don't touch INIT. Child tells parent there is a problem if self.parent_id!=1 and self.parent_id!=mypid: self.ShowSignalMessage(f'Signal parent: {self.parent_id}') os.kill(self.parent_id,2) - # If I am the parent, give the children time to wrap it up - if current_process().name!='MainProcess': - c=0 - while c<3: - active_children() - ElasticSleep(1) - c+=1 - self.ZombieHunter() - # Shut it all down - if self.parent_id==mypid and len(parent.children())>1: + if self.parent_id==mypid and self.IsParent and nChildren>0: self.ShowSignalMessage(f'Exiting parent: {mypid}') - elif self.parent_id==mypid: - self.ShowSignalMessage(f'Exiting self: {mypid}') - else: + elif self.parent_id==mypid and self.IsChild: self.ShowSignalMessage(f'Exiting child: {mypid}') + else: + self.ShowSignalMessage(f'Exiting self: {mypid}') os.kill(mypid,9) - def SetLog(self,Log=None): - self.Log=Log - - def Critical(self,IsCrit=False): - # Check is there is a trigger. If a signal is triggered, safely exit BEFORE critical event - if IsCrit==True and self.critical==False: - for sig in signal.valid_signals(): - if self.triggered[sig]==True: - self.SafeExit() - self.critical=IsCrit - - # There HAS TO BE better ways then this brute force way of cleaning up after multiprocessing for crashed/exited/dead child - # processes. + # Signal handler for child process exit - def ZombieHunter(self): - init=psutil.Process(1) - for child in init.children(): - cmd=child.cmdline() - for i in range(len(cmd)): - if 'from multiprocessing.resource_tracker import main;main' in cmd[i] \ - or 'from multiprocessing.spawn import spawn_main; spawn_main' in cmd[i]: - os.kill(child.pid,9) + def SignalChild(self,signum,frame): + try: + # Check if any child processes have exited + pid, exit_code=os.waitpid(-1,os.WNOHANG) + except: + pass - def mpCleaner(self,signum,frame): - active_children() - self.ZombieHunter() - signal.alarm(13) + # We received a signal, process it. def ProcessSignal(self,signum,frame): self.ShowSignalMessage(f'Interceptor Signal: {signum} In Critical: {self.critical}') @@ -137,10 +133,14 @@ def ProcessSignal(self,signum,frame): if self.critical==False: self.SafeExit() + # Force reset all signal statess + def ResetSignals(self): for sig in signal.valid_signals(): self.triggered[sig]=False + # Restore the original signal handlers + def RestoreOriginalSignals(self): for sig in signal.valid_signals(): try: @@ -148,9 +148,30 @@ def RestoreOriginalSignals(self): except: pass + # Ignore all signals, for child process + + def IgnoreSignals(self): + for sig in signal.valid_signals(): + try: + signal.signal(sig,signal.SIG_IGN) + except: + pass + + # Has a single signal been triggered? + def Triggered(self,signum): return self.triggered[signum] + # Has ANY supported signal been triggered? + + def AnyTriggered(self): + for sig in signal.valid_signals(): + if self.triggered[sig]==True: + return True + return False + + # A safe way to exit the program is it is not in a critical situation. + def SafeExit(self,now=False): for sig in signal.valid_signals(): if (self.triggered[sig]==True and self.critical==False) or now==True: @@ -158,6 +179,55 @@ def SafeExit(self,now=False): sig=9 self.SignalInterrupt(sig) + # Set whether we are entering a critical area where signals will be ignored, like writing file data. + + def Critical(self,IsCrit=False): + # Check is there is a trigger. If a signal is triggered, safely exit BEFORE critical event + if IsCrit==True and self.critical==False and self.AnyTriggered()==True: + self.SafeExit() + self.critical=IsCrit + + # Return the number of child processes. Needed for large lists of tasks to be completed. + + def GetChildren(self): + parent = psutil.Process(self.parent_id) + return len(parent.children(recursive=False)) + + # Crude way to tell if this process or function is a child or the parent + + def WhoAmI(self): + if self.IsParent: + return "Parent" + elif self.IsChild: + return "Child" + else: + return "Orphan" + + # Very simple multiprocessing methodology. + + def StartProcess(self,func,args=None,kwargs=None): + if args==None: + args=[] + if kwargs==None: + kwargs={} + + pid=os.fork() + if pid==0: + self.IsParent=False + self.IsChild=True + self.IgoreSignals() + # Child process + try: + # Call the function with the provided arguments + func(*args, **kwargs) + sys.exit(0) + except Exception as e: + # Handle child process error + sys.exit(1) + + # Parent process + return pid + # Reusable file locks, using atomic operations # NOT suitable for distributed systems or # Windows. Linux ONLY @@ -862,3 +932,7 @@ def purge(self): except: self.fw.Unlock() +### +### End of module +### + diff --git a/Base/Library/JackrabbitRelay.py b/Base/Library/JackrabbitRelay.py index fc131fd..12aa79a 100755 --- a/Base/Library/JackrabbitRelay.py +++ b/Base/Library/JackrabbitRelay.py @@ -104,7 +104,7 @@ def Success(self,f,s): class JackrabbitRelay: def __init__(self,framework=None,payload=None,exchange=None,account=None,asset=None,secondary=None,NoIdentityVerification=False): # All the default locations - self.Version="0.0.0.1.340" + self.Version="0.0.0.1.370" self.BaseDirectory='/home/JackrabbitRelay2/Base' self.ConfigDirectory='/home/JackrabbitRelay2/Config' self.DataDirectory="/home/JackrabbitRelay2/Data" @@ -412,7 +412,9 @@ def TradingViewRemap(self): # Process and validate the order payload - def ProcessPayload(self): + def ProcessPayload(self,NewPayload=None): + if NewPayload!=None: + self.Payload=NewPayload if self.Payload==None or self.Payload.strip()=='': self.JRLog.Error('Processing Payload','Empty payload') @@ -744,3 +746,41 @@ def MakeConditionalOrder(self,id,Order): def WriteLedger(self,**kwargs): self.Results=self.Broker.WriteLedger(**kwargs,LedgerDirectory=self.LedgerDirectory) self.EnforceRateLimit() + + # See if an order is already in Oliver Twist for Exchange/Account/Pair. This is to allow ONLY ONE order at a time. + + def OliverTwistOneShot(self,CompareOrder): + fList=[self.DataDirectory+'/OliverTwist.Conditional.Receiver',self.DataDirectory+'/OliverTwist.Conditional.Storehouse'] + orphanLock=JRRsupport.Locker("OliverTwist") + + orphanLock.Lock() + for fn in fList: + if os.path.exists(fn): + buffer=JRRsupport.ReadFile(fn) + if buffer!=None and buffer!='': + Orphans=buffer.split('\n') + for Entry in Orphans: + # Force set InMotion to False + Entry=Entry.strip() + if Entry=='': + continue + # Break down entry and set up memory locker + try: + Orphan=json.loads(Entry) + except: + continue + + if 'Order' in Orphan: + if type(Orphan['Order'])==str: + order=json.loads(Orphan['Order']) + else: + order=Orphan['Order'] + if CompareOrder['Exchange']==order['Exchange'] \ + and CompareOrder['Account']==order['Account'] \ + and CompareOrder['Asset']==order['Asset']: + orphanLock.Unlock() + return True + + # Not Found + orphanLock.Unlock() + return False diff --git a/Base/OANDA-PlaceOrder b/Base/OANDA-PlaceOrder index eeddf25..2da917a 100755 --- a/Base/OANDA-PlaceOrder +++ b/Base/OANDA-PlaceOrder @@ -111,6 +111,13 @@ def main(): if ("Conditional" in relay.Active or "Conditional" in relay.Order) and relay.Order['Action']!='buy': relay.JRLog.Error("Conditional", f"{relay.Order['Action']}: ignored") + # Check for OneShot situation + + if ("Conditional" in relay.Active or "Conditional" in relay.Order) \ + and ("ConditionalOneShot" in relay.Active or "ConditionalOneShot" in relay.Order): + if relay.OliverTwistOneShot(relay.Order)==True: + relay.JRLog.Error("Conditional OneShot", f"{relay.Order['Exchange']}/{relay.Order['Account']}/{relay.Order['Asset']} already being managed by OliverTwist ") + # Now lets get down to business. The processed order is in: # relay.Order diff --git a/Base/Orphan.ccxt b/Base/Orphan.ccxt index 53ef2d1..19d97fd 100755 --- a/Base/Orphan.ccxt +++ b/Base/Orphan.ccxt @@ -14,20 +14,45 @@ import time import JRRsupport import JackrabbitRelay as JRR +# Timeout before Locker auto-deletes this order result + +OliverTwistTimeout=(15*60) + +# Write the result to Locker memory so parent knows we are finished. + +def FinishOrphan(Key,lID,mID,State): + # Get the lock read and set up the memory key. Locker doesn't care and which class this order belongs to. OliverTwist will + # match the ID to the right class list, orphan or conditional. + + OliverTwistLock=JRRsupport.Locker("OliverTwist",ID=lID) + Memory=JRRsupport.Locker(Key,ID=mID) + + OliverTwistLock.Lock() + + State=State.lower() + + if State!='delete': + # Return this order to a waiting state + Memory.Put(OliverTwistTimeout*100,"Waiting") + elif State=='delete': + # This order has been processed and needs to be removed from the system. + Memory.Put(OliverTwistTimeout*100,"Delete") + + OliverTwistLock.Unlock() + + # We're done. This child has completed its task + sys.exit(0) + +### +### Main driver +### + def main(): data=sys.stdin.read().strip() try: Orphan=json.loads(data,strict=False) except: - # Emergency falthru to prevent damage to the orphan list. - rData={} - rData['Key']='Corrupted' - rData['lID']='Corrupted' - rData['mID']='Corrupted' - rData['Status']='open' - print(json.dumps(rData)) - sys.stdout.flush() - sys.exit(0) + FinishOrphan(Orphan['Key'],Orphan['lID'],Orphan['mID'],'open') # Use Relay to process and validate the order. Order MUST be a JSON string. @@ -45,27 +70,11 @@ def main(): #relay.JRLog.Write(f"Orders: {len(openOrders)}",stdOut=False) for cur in openOrders: if cur['id']==id: - rData={} - rData['Key']=Orphan['Key'] - rData['lID']=Orphan['lID'] - rData['mID']=Orphan['mID'] - rData['Class']=Orphan['Class'] - rData['Status']=cur['status'] - print(json.dumps(rData)) - sys.stdout.flush() - #relay.JRLog.Write(f"{id}: {cur['status']}",stdOut=False) - sys.exit(0) + FinishOrphan(Orphan['Key'],Orphan['lID'],Orphan['mID'],cur['state']) # Order must be closed - rData={} - rData['Key']=Orphan['Key'] - rData['lID']=Orphan['lID'] - rData['mID']=Orphan['mID'] - rData['Class']=Orphan['Class'] - rData['Status']='Delete' - print(json.dumps(rData)) - sys.stdout.flush() relay.WriteLedger(Order=Orphan,Response=None) + FinishOrphan(Orphan['Key'],Orphan['lID'],Orphan['mID'],'Delete') if __name__ == '__main__': main() diff --git a/Base/Orphan.oanda b/Base/Orphan.oanda index cf4544d..3ce6506 100755 --- a/Base/Orphan.oanda +++ b/Base/Orphan.oanda @@ -14,6 +14,39 @@ import time import JRRsupport import JackrabbitRelay as JRR +# Timeout before Locker auto-deletes this order result + +OliverTwistTimeout=(15*60) + +# Write the result to Locker memory so parent knows we are finished. + +def FinishOrphan(Key,lID,mID,State): + # Get the lock read and set up the memory key. Locker doesn't care and which class this order belongs to. OliverTwist will + # match the ID to the right class list, orphan or conditional. + + OliverTwistLock=JRRsupport.Locker("OliverTwist",ID=lID) + Memory=JRRsupport.Locker(Key,ID=mID) + + OliverTwistLock.Lock() + + State=State.lower() + + if State!='delete': + # Return this order to a waiting state + Memory.Put(OliverTwistTimeout,"Waiting") + elif State=='delete': + # This order has been processed and needs to be removed from the system. + Memory.Put(OliverTwistTimeout,"Delete") + + OliverTwistLock.Unlock() + + # We're done. This child has completed its task + sys.exit(0) + +### +### Main driver +### + def main(): data=sys.stdin.read().strip() Orphan=json.loads(data) @@ -34,27 +67,11 @@ def main(): openOrders=relay.GetOpenOrders(symbol=relay.Order['Asset']) for cur in openOrders: if cur['id']==id: - rData={} - rData['Key']=Orphan['Key'] - rData['lID']=Orphan['lID'] - rData['mID']=Orphan['mID'] - rData['Class']=Orphan['Class'] - rData['Status']=cur['state'] - print(json.dumps(rData)) - sys.stdout.flush() - #relay.JRLog.Write(f"{id}: {cur['state']}",stdOut=False) - sys.exit(0) + FinishOrphan(Orphan['Key'],Orphan['lID'],Orphan['mID'],cur['state']) # Order must be closed - rData={} - rData['Key']=Orphan['Key'] - rData['lID']=Orphan['lID'] - rData['mID']=Orphan['mID'] - rData['Class']=Orphan['Class'] - rData['Status']='Delete' - print(json.dumps(rData)) - sys.stdout.flush() relay.WriteLedger(Order=Orphan,Response=None) + FinishOrphan(Orphan['Key'],Orphan['lID'],Orphan['mID'],'Delete') if __name__ == '__main__': main() diff --git a/Extras/CodeProofs/BuyEveryMinute b/Extras/CodeProofs/BuyEveryMinute new file mode 100755 index 0000000..f19c854 --- /dev/null +++ b/Extras/CodeProofs/BuyEveryMinute @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Jackrabbit Relay +# 2021 Copyright © Robert APM Darin +# All rights reserved unconditionally. + +import sys +sys.path.append('/home/GitHub/JackrabbitRelay/Base/Library') +import os +import json +import time + +import JRRsupport +import JackrabbitRelay as JRR + +NewOrder='{"Recipe":"#Tester","Action":"Buy","Exchange":"phemex","Market":"Spot","Account":"Sandbox","Asset":"TRX/USDT","USD":"20","OrderType":"Market","Link":"https://www.tradingview.com/chart/L0ejIBsJ/","Comment":"Conditional order entries","Conditional":"Yes","ConditionalOneShot":"Yes","Direction":"Long","SellAction":"Sell","TakeProfit":"1%","StopLoss":"10%" }' + +# Get the order ID. If there isn't an ID, the order FAILED. + +def GetOrderID(res): + if res.find('Order Confirmation ID')>-1: + s=res.find('ID:')+4 + for e in range(s,len(res)): + if res[e]=='\n': + break + oid=res[s:e] + + return oid + return None + +### +### Main driver +### + +def main(): + relay=JRR.JackrabbitRelay(payload=NewOrder,NoIdentityVerification=True) + relay.Order['Identity']=relay.Active['Identity'] + + while True: + # Feed the new order to Relay + result=relay.SendWebhook(relay.Order) + print(result) + oid=GetOrderID(result) + if oid!=None: + bal=relay.GetBalance(Base=relay.Order['Asset'].split('/')[0]) + print(oid,bal) + JRRsupport.ElasticSleep(60) + +if __name__ == '__main__': + main() diff --git a/Extras/CodeProofs/ForkTest b/Extras/CodeProofs/ForkTest new file mode 100755 index 0000000..b584838 --- /dev/null +++ b/Extras/CodeProofs/ForkTest @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Jackrabbit Relay +# 2021 Copyright © Robert APM Darin +# All rights reserved unconditionally. + +import sys +sys.path.append('/home/GitHub/JackrabbitRelay/Base/Library') +import os +import time +import random + +import JRRsupport + +if __name__=='__main__': + interceptor=JRRsupport.SignalInterceptor(IsMain=True) +else: + interceptor=JRRsupport.SignalInterceptor(IsMain=False) + +def RandoSleeper(): + t=random.randint(1,20) + print(f"{os.getppid()}/{os.getpid()}: {interceptor.WhoAmI()} sleeping {t} seconds") + time.sleep(t) + +def main(): + print(f"Parent: {os.getpid()}, {interceptor.AnyTriggered()}") + RandoSleeper() + + for i in range(100): + interceptor.StartProcess(RandoSleeper) + while interceptor.GetChildren()>10: + l=interceptor.GetChildren() + print(f"{i}/{l} {interceptor.AnyTriggered()}") + if l==0: + break + time.sleep(1) + + while True: + l=interceptor.GetChildren() + print(l,interceptor.AnyTriggered()) + if l==0: + break + time.sleep(1) + +### +### Main driver +### + +if __name__=='__main__': + main() + +### +### End +### + diff --git a/Extras/ListMarkets b/Extras/ListMarkets index a35088e..08de251 100755 --- a/Extras/ListMarkets +++ b/Extras/ListMarkets @@ -7,6 +7,12 @@ import os import JackrabbitRelay as JRR +def GetRatio(decimal): + ratio = 1 / decimal + return f"{int(ratio):4}:1" + +# Example usage + def main(): relay=JRR.JackrabbitRelay() if relay.GetArgsLen() > 2: @@ -62,7 +68,8 @@ def main(): id=relay.Markets[pair]['id'] print(f"{pair:30} {id:30} {marketType:8} {ticker['Ask']:18.8f} {ticker['Spread']:18.8f} {ticker['Bid']:18.8f}") elif relay.Framework=='oanda': - print(f"{pair:30} {ticker['Ask']:9.5f} {ticker['Spread']:9.5f} {ticker['Bid']:9.5f}") + mr=GetRatio(float(relay.Markets[pair]['marginRate'])) + print(f"{pair:30} {mr} {ticker['Ask']:9.5f} {ticker['Spread']:9.5f} {ticker['Bid']:9.5f}") else: print("Unrecognized framework") sys.exit(0) diff --git a/Extras/otcWatch b/Extras/otcWatch index ca06c1d..aa3d881 100755 --- a/Extras/otcWatch +++ b/Extras/otcWatch @@ -65,6 +65,10 @@ def main(): print("Exchange/Broker and account must be provided.") sys.exit(0) + srch=None + if len(sys.argv)>3: + srch=sys.argv[3] + OliverTwistLock.Lock() if os.path.exists(ConditionalStorehouse): buffer=JRRsupport.ReadFile(ConditionalStorehouse) @@ -92,13 +96,19 @@ def main(): if Order['Exchange']!=exchange or Order['Account']!=account: continue + if srch!=None and Order['Asset']!=srch: + continue + price=0 amount=0 # Cache and remember these data points. Exchanges like phemex will draw the program into oblivion otherwise from # the rtelimits. This only works because we are filtering based upon exchange and account. - relay=JRR.JackrabbitRelay(framework=Data['Framework'],payload=json.dumps(Order)) + if relay==None: + relay=JRR.JackrabbitRelay(framework=Data['Framework'],payload=json.dumps(Order),NoIdentityVerification=True) + else: + relay.ProcessPayload(NewPayload=json.dumps(Order)) if oldAsset==None or oldAsset!=relay.Order['Asset']: ticker=relay.GetTicker(symbol=relay.Order['Asset']) oldAsset=relay.Order['Asset'] @@ -106,7 +116,11 @@ def main(): # Handle each framework type if Data['Framework']=='ccxt': - Resp=Data['Response'] + id=Data['ID'] + if type(Data['Response'])==str: + Resp=json.loads(Data['Response']) + else: + Resp=Data['Response'] Detail=Resp['Details'] price=Detail['price'] amount=Detail['amount'] @@ -120,7 +134,7 @@ def main(): tp=round(CalculatePriceExit(Order,'TakeProfit',dir,price),8) sl=round(CalculatePriceExit(Order,'StopLoss',dir,price),8) - print(f"{Order['Asset']:20} {sl:.8f} {tprice:.8f}({price:.8f}) {tp:.8f} {amount:.8f} {dir}") + print(f"{id} {Order['Asset']:14} {sl:.8f} {tprice:.8f}({price:.8f}) {tp:.8f} {amount:.8f} {dir}") elif Data['Framework']=='oanda': Resp=Data['Response'] id=Data['ID']