Correct template accept ratio sorting to account for added block height
[bitcoin:eloipool.git] / merklemaker.py
1 # Eloipool - Python Bitcoin pool server
2 # Copyright (C) 2011-2013  Luke Dashjr <luke-jr+eloipool@utopios.org>
3 #
4 # This program is free software: you can redistribute it and/or modify
5 # it under the terms of the GNU Affero General Public License as
6 # published by the Free Software Foundation, either version 3 of the
7 # License, or (at your option) any later version.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12 # GNU Affero General Public License for more details.
13 #
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
16
17 from binascii import b2a_hex
18 import bitcoin.script
19 from bitcoin.script import countSigOps
20 from bitcoin.txn import Txn
21 from bitcoin.varlen import varlenEncode, varlenDecode
22 from collections import deque
23 from copy import deepcopy
24 from queue import Queue
25 import jsonrpc
26 import logging
27 from math import log
28 from merkletree import MerkleTree
29 import socket
30 from struct import pack
31 import threading
32 from time import sleep, time
33 import traceback
34
35 _makeCoinbase = [0, 0]
36 _filecounter = 0
37
38 def MakeBlockHeader(MRD):
39         (merkleRoot, merkleTree, coinbase, prevBlock, bits) = MRD[:5]
40         timestamp = pack('<L', int(time()))
41         hdr = b'\2\0\0\0' + prevBlock + merkleRoot + timestamp + bits + b'iolE'
42         return hdr
43
44 def assembleBlock(blkhdr, txlist):
45         payload = blkhdr
46         payload += varlenEncode(len(txlist))
47         for tx in txlist:
48                 payload += tx.data
49         return payload
50
51 class merkleMaker(threading.Thread):
52         GBTCaps = [
53                 'coinbasevalue',
54                 'coinbase/append',
55                 'coinbase',
56                 'generation',
57                 'time',
58                 'transactions/remove',
59                 'prevblock',
60         ]
61         GBTReq = {
62                 'capabilities': GBTCaps,
63         }
64         GMPReq = {
65                 'capabilities': GBTCaps,
66                 'tx': 'obj',
67         }
68         
69         def __init__(self, *a, **k):
70                 super().__init__(*a, **k)
71                 self.daemon = True
72                 self.logger = logging.getLogger('merkleMaker')
73                 self.CoinbasePrefix = b''
74                 self.CoinbaseAux = {}
75                 self.isOverflowed = False
76                 self.lastWarning = {}
77                 self.MinimumTxnUpdateWait = 5
78                 self.overflowed = 0
79                 self.DifficultyChangeMod = 2016
80                 self.MinimumTemplateAcceptanceRatio = 0
81                 self.MinimumTemplateScore = 1
82                 self.currentBlock = (None, None, None)
83                 self.lastBlock = (None, None, None)
84                 self.SubsidyAlgo = lambda height: 5000000000 >> (height // 210000)
85         
86         def _prepare(self):
87                 self.TemplateSources = list(getattr(self, 'TemplateSources', ()))
88                 self.TemplateChecks = list(getattr(self, 'TemplateChecks', ()))
89                 if getattr(self, 'BlockSubmissions', None) is None:
90                         self.BlockSubmissions = ()
91                 if hasattr(self, 'UpstreamURI'):
92                         self.TemplateSources.append({
93                                 'name': 'UpstreamURI',
94                                 'uri': self.UpstreamURI,
95                         })
96                 URI2Name = {}
97                 Name2URI = {}
98                 for a in (self.TemplateSources + self.TemplateChecks + list(self.BlockSubmissions)):
99                         if not ('name' in a and 'uri' in a):
100                                 continue
101                         URI2Name.setdefault(a['uri'], a['name'])
102                         Name2URI.setdefault(a['name'], a['uri'])
103                 def URINamePair(a, defname):
104                         if 'name' not in a:
105                                 a['name'] = URI2Name.get(a['uri'], defname)
106                         elif 'uri' not in a:
107                                 a['uri'] = Name2URI[a['name']]
108                 _URI2Access = {}
109                 def URI2Access(uri):
110                         if uri not in _URI2Access:
111                                 access = jsonrpc.ServiceProxy(uri)
112                                 access.OldGMP = False
113                                 _URI2Access[uri] = access
114                         return _URI2Access[uri]
115                 LeveledTS = {}
116                 for i in range(len(self.TemplateSources)):
117                         TS = self.TemplateSources[i]
118                         URINamePair(TS, 'TemplateSources[%u]' % (i,))
119                         TS.setdefault('priority', 0)
120                         TS.setdefault('weight', 1)
121                         TS['access'] = URI2Access(TS['uri'])
122                         LeveledTS.setdefault(TS['priority'], []).append(TS)
123                 LeveledTS = tuple(x[1] for x in sorted(LeveledTS.items()))
124                 self.TemplateSources = LeveledTS
125                 for i in range(len(self.TemplateChecks)):
126                         TC = self.TemplateChecks[i]
127                         URINamePair(TC, 'TemplateChecks[%u]' % (i,))
128                         TC.setdefault('unanimous', False)
129                         TC.setdefault('weight', 1)
130                         TC['access'] = URI2Access(TC['uri'])
131                 for i in range(len(getattr(self, 'BlockSubmissions', ()))):
132                         BS = self.BlockSubmissions[i]
133                         URINamePair(BS, 'BlockSubmissions[%u]' % (i,))
134                         BS['access'] = URI2Access(BS['uri'])
135                 
136                 self.ready = False
137                 self.readyCV = threading.Condition()
138                 
139                 self.currentMerkleTree = None
140                 self.merkleRoots = deque(maxlen=self.WorkQueueSizeRegular[1])
141                 self.LowestMerkleRoots = self.WorkQueueSizeRegular[1]
142                 
143                 if not hasattr(self, 'WorkQueueSizeClear'):
144                         self.WorkQueueSizeClear = self.WorkQueueSizeLongpoll
145                 self._MaxClearSize = max(self.WorkQueueSizeClear[1], self.WorkQueueSizeLongpoll[1])
146                 self.clearMerkleRoots = Queue(self._MaxClearSize)
147                 self.LowestClearMerkleRoots = self.WorkQueueSizeClear[1]
148                 self.nextMerkleRoots = Queue(self._MaxClearSize)
149                 
150                 if not hasattr(self, 'WarningDelay'):
151                         self.WarningDelay = max(15, self.MinimumTxnUpdateWait * 2)
152                 if not hasattr(self, 'WarningDelayTxnLongpoll'):
153                         self.WarningDelayTxnLongpoll = self.WarningDelay
154                 if not hasattr(self, 'WarningDelayMerkleUpdate'):
155                         self.WarningDelayMerkleUpdate = self.WarningDelay
156                 
157                 self.lastMerkleUpdate = 0
158                 self.nextMerkleUpdate = 0
159         
160         def createClearMerkleTree(self, height):
161                 subsidy = self.SubsidyAlgo(height)
162                 cbtxn = self.makeCoinbaseTxn(subsidy, False)
163                 cbtxn.assemble()
164                 return MerkleTree([cbtxn])
165         
166         def updateBlock(self, newBlock, height = None, bits = None, _HBH = None):
167                 if newBlock == self.currentBlock[0]:
168                         if height in (None, self.currentBlock[1]) and bits in (None, self.currentBlock[2]):
169                                 return
170                         if not self.currentBlock[2] is None:
171                                 self.logger.error('Was working on block with wrong specs: %s (height: %d->%d; bits: %s->%s' % (
172                                         b2a_hex(newBlock[::-1]).decode('utf8'),
173                                         self.currentBlock[1],
174                                         height,
175                                         b2a_hex(self.currentBlock[2][::-1]).decode('utf8'),
176                                         b2a_hex(bits[::-1]).decode('utf8'),
177                                 ))
178                                 if self.needMerkle == 1:
179                                         self.needMerkle = False
180                                 self.onBlockUpdate()
181                 
182                 # Old block is invalid
183                 if self.currentBlock[0] != newBlock:
184                         self.lastBlock = self.currentBlock
185                 
186                 lastHeight = self.currentBlock[1]
187                 if height is None:
188                         height = self.currentBlock[1] + 1
189                 if bits is None:
190                         if height % self.DifficultyChangeMod == 1 or self.currentBlock[2] is None:
191                                 self.logger.warning('New block: %s (height %d; bits: UNKNOWN)' % (b2a_hex(newBlock[::-1]).decode('utf8'), height))
192                                 # Pretend to be 1 lower height, so we possibly retain nextMerkleRoots
193                                 self.currentBlock = (None, height - 1, None)
194                                 OCMR = self.clearMerkleRoots
195                                 self.clearMerkleRoots = Queue(self.WorkQueueSizeClear[1])
196                                 if OCMR.empty():
197                                         OCMR.put(None)
198                                 self.merkleRoots.clear()
199                                 self.ready = False
200                                 return
201                         else:
202                                 bits = self.currentBlock[2]
203                 
204                 if _HBH is None:
205                         _HBH = (b2a_hex(newBlock[::-1]).decode('utf8'), b2a_hex(bits[::-1]).decode('utf8'))
206                 self.logger.info('New block: %s (height: %d; bits: %s)' % (_HBH[0], height, _HBH[1]))
207                 self.currentBlock = (newBlock, height, bits)
208                 
209                 if lastHeight != height:
210                         # TODO: Perhaps reuse clear merkle trees more intelligently
211                         OCMR = self.clearMerkleRoots
212                         if lastHeight == height - 1:
213                                 self.curClearMerkleTree = self.nextMerkleTree
214                                 self.clearMerkleRoots = self.nextMerkleRoots
215                                 self.logger.debug('Adopting next-height clear merkleroots :)')
216                         else:
217                                 if lastHeight:
218                                         self.logger.warning('Change from height %d->%d; no longpoll merkleroots available!' % (lastHeight, height))
219                                 self.curClearMerkleTree = self.createClearMerkleTree(height)
220                                 self.clearMerkleRoots = Queue(self.WorkQueueSizeClear[1])
221                         if OCMR.empty():
222                                 OCMR.put(None)
223                         self.nextMerkleTree = self.createClearMerkleTree(height + 1)
224                         self.nextMerkleRoots = Queue(self._MaxClearSize)
225                 else:
226                         self.logger.debug('Already using clear merkleroots for this height')
227                 self.currentMerkleTree = self.curClearMerkleTree
228                 self.merkleRoots.clear()
229                 
230                 if not self.ready:
231                         self.ready = True
232                         with self.readyCV:
233                                 self.readyCV.notify_all()
234                 
235                 self.needMerkle = 2
236                 self.onBlockChange()
237         
238         def _trimBlock(self, MP, txnlist, txninfo, floodn, msgf):
239                 fee = txninfo[-1].get('fee', None)
240                 if fee is None:
241                         raise self._floodCritical(now, floodn, doin=msgf('fees unknown'))
242                 if fee:
243                         # FIXME: coinbasevalue is *not* guaranteed to exist here
244                         MP['coinbasevalue'] -= fee
245                 
246                 txnlist[-1:] = ()
247                 txninfo[-1:] = ()
248                 
249                 return True
250         
251         # Aggressive "Power Of Two": Remove transactions even with fees to reach our goal
252         def _APOT(self, txninfopot, MP, POTInfo):
253                 feeTxnsTrimmed = 0
254                 feesTrimmed = 0
255                 for txn in txninfopot:
256                         if txn.get('fee') is None:
257                                 self._floodWarning(now, 'APOT-No-Fees', doin='Upstream didn\'t provide fee information required for aggressive POT', logf=self.logger.info)
258                                 return
259                         if not txn['fee']:
260                                 continue
261                         feesTrimmed += txn['fee']
262                         feeTxnsTrimmed += 1
263                 MP['coinbasevalue'] -= feesTrimmed
264                 
265                 POTInfo[2] = [feeTxnsTrimmed, feesTrimmed]
266                 self._floodWarning(now, 'POT-Trimming-Fees', doin='Aggressive POT trimming %d transactions with %d.%08d BTC total fees' % (feeTxnsTrimmed, feesTrimmed//100000000, feesTrimmed % 100000000), logf=self.logger.debug)
267                 
268                 return True
269         
270         def _makeBlockSafe(self, MP, txnlist, txninfo):
271                 blocksize = sum(map(len, txnlist)) + 80
272                 while blocksize > 934464:  # 1 "MB" limit - 64 KB breathing room
273                         txnsize = len(txnlist[-1])
274                         self._trimBlock(MP, txnlist, txninfo, 'SizeLimit', lambda x: 'Making blocks over 1 MB size limit (%d bytes; %s)' % (blocksize, x))
275                         blocksize -= txnsize
276                 
277                 # NOTE: This check doesn't work at all without BIP22 transaction obj format
278                 blocksigops = sum(a.get('sigops', 0) for a in txninfo)
279                 while blocksigops > 19488:  # 20k limit - 0x200 breathing room
280                         txnsigops = txninfo[-1]['sigops']
281                         self._trimBlock(MP, txnlist, txninfo, 'SigOpLimit', lambda x: 'Making blocks over 20k SigOp limit (%d; %s)' % (blocksigops, x))
282                         blocksigops -= txnsigops
283                 
284                 # Aim to produce blocks with "Power Of Two" transaction counts
285                 # This helps avoid any chance of someone abusing CVE-2012-2459 with them
286                 POTMode = getattr(self, 'POT', 1)
287                 txncount = len(txnlist) + 1
288                 if POTMode:
289                         feetxncount = txncount
290                         for i in range(txncount - 2, -1, -1):
291                                 if 'fee' not in txninfo[i] or txninfo[i]['fee']:
292                                         break
293                                 feetxncount -= 1
294                         
295                         if getattr(self, 'Greedy', None):
296                                 # Aim to cut off extra zero-fee transactions on the end
297                                 # NOTE: not cutting out ones intermixed, in case of dependencies
298                                 idealtxncount = feetxncount
299                         else:
300                                 idealtxncount = txncount
301                         
302                         pot = 2**int(log(idealtxncount, 2))
303                         POTInfo = MP['POTInfo'] = [[idealtxncount, feetxncount, txncount], [pot, None], None]
304                         if pot < idealtxncount:
305                                 if pot * 2 <= txncount:
306                                         pot *= 2
307                                 elif pot >= feetxncount:
308                                         pass
309                                 elif POTMode > 1:
310                                         if self._APOT(txninfo[pot-1:], MP, POTInfo):
311                                                 # Trimmed even transactions with fees
312                                                 pass
313                                         else:
314                                                 pot = idealtxncount
315                                                 self._floodWarning(now, 'Non-POT', doin='Making merkle tree with %d transactions (ideal: %d; max: %d)' % (pot, idealtxncount, txncount))
316                                 else:
317                                         pot = idealtxncount
318                         POTInfo[1][1] = pot
319                         pot -= 1
320                         txnlist[pot:] = ()
321                         txninfo[pot:] = ()
322         
323         def _CallGBT(self, TS):
324                 access = TS['access']
325                 self.logger.debug('Requesting new template from \'%s\'' % (TS['name'],))
326                 try:
327                         # First, try BIP 22 standard getblocktemplate :)
328                         MP = access.getblocktemplate(self.GBTReq)
329                         access.OldGMP = False
330                 except:
331                         try:
332                                 # Failing that, give BIP 22 draft (2012-02 through 2012-07) getmemorypool a chance
333                                 MP = access.getmemorypool(self.GMPReq)
334                         except:
335                                 try:
336                                         # Finally, fall back to bitcoind 0.5/0.6 getmemorypool
337                                         MP = access.getmemorypool()
338                                 except:
339                                         MP = False
340                         if MP is False:
341                                 # This way, we get the error from the BIP22 call if the old one fails too
342                                 raise
343                         
344                         # Pre-BIP22 server (bitcoind <0.7 or Eloipool <20120513)
345                         if not access.OldGMP:
346                                 access.OldGMP = True
347                                 self.logger.warning('Upstream \'%s\' is not BIP 22 compatible' % (TS['name'],))
348                 
349                 return MP
350         
351         def _ProcessGBT(self, MP, TS = None):
352                 oMP = MP
353                 MP = deepcopy(MP)
354                 
355                 prevBlock = bytes.fromhex(MP['previousblockhash'])[::-1]
356                 if 'height' not in MP:
357                         MP['height'] = TS['access'].getinfo()['blocks'] + 1
358                 height = MP['height']
359                 bits = bytes.fromhex(MP['bits'])[::-1]
360                 (MP['_bits'], MP['_prevBlock']) = (bits, prevBlock)
361                 if (prevBlock, height, bits) != self.currentBlock and (self.currentBlock[1] is None or height > self.currentBlock[1]):
362                         self.updateBlock(prevBlock, height, bits, _HBH=(MP['previousblockhash'], MP['bits']))
363                 
364                 txnlist = MP['transactions']
365                 if len(txnlist) and isinstance(txnlist[0], dict):
366                         txninfo = txnlist
367                         txnlist = tuple(a['data'] for a in txnlist)
368                 elif 'transactionfees' in MP:
369                         # Backward compatibility with pre-BIP22 gmp_fees branch
370                         txninfo = [{'fee':a} for a in MP['transactionfees']]
371                 else:
372                         # Backward compatibility with pre-BIP22 hex-only (bitcoind <0.7, Eloipool <future)
373                         txninfo = [{}] * len(txnlist)
374                 # TODO: cache Txn or at least txid from previous merkle roots?
375                 txnlist = [a for a in map(bytes.fromhex, txnlist)]
376                 
377                 self._makeBlockSafe(MP, txnlist, txninfo)
378                 
379                 cbtxn = self.makeCoinbaseTxn(MP['coinbasevalue'], prevBlockHex = MP['previousblockhash'])
380                 cbtxn.setCoinbase(b'\0\0')
381                 cbtxn.assemble()
382                 txnlist.insert(0, cbtxn.data)
383                 txninfo.insert(0, {
384                 })
385                 
386                 txnlist = [a for a in map(Txn, txnlist[1:])]
387                 txnlist.insert(0, cbtxn)
388                 txnlist = list(txnlist)
389                 newMerkleTree = MerkleTree(txnlist)
390                 newMerkleTree.POTInfo = MP.get('POTInfo')
391                 newMerkleTree.MP = MP
392                 newMerkleTree.oMP = oMP
393                 
394                 return newMerkleTree
395         
396         def _CheckTemplate(self, newMerkleTree, TS):
397                 TCList = self.TemplateChecks
398                 if not TCList:
399                         if 'proposal' not in newMerkleTree.oMP.get('capabilities', ()):
400                                 return (None, None)
401                         TCList = (
402                                 {
403                                         'name': TS['name'],
404                                         'access': TS['access'],
405                                         'unanimous': True,
406                                         'weight': 1,
407                                 },
408                         )
409                 
410                 MP = newMerkleTree.MP
411                 (prevBlock, height, bits) = (MP['_prevBlock'], MP['height'], MP['_bits'])
412                 txnlist = newMerkleTree.data
413                 cbtxn = txnlist[0]
414                 
415                 coinbase = self.makeCoinbase(height=height)
416                 cbtxn.setCoinbase(coinbase)
417                 cbtxn.assemble()
418                 merkleRoot = newMerkleTree.merkleRoot()
419                 MRD = (merkleRoot, newMerkleTree, coinbase, prevBlock, bits)
420                 blkhdr = MakeBlockHeader(MRD)
421                 data = assembleBlock(blkhdr, txnlist)
422                 ProposeReq = {
423                         "mode": "proposal",
424                         "data": b2a_hex(data).decode('utf8'),
425                 }
426                 
427                 AcceptedScore = 0
428                 RejectedScore = 0
429                 Rejections = {}
430                 ProposalErrors = {}
431                 for TC in TCList:
432                         caccess = TC['access']
433                         try:
434                                 propose = caccess.getblocktemplate(ProposeReq)
435                         except (socket.error, ValueError) as e:
436                                 self.logger.error('Upstream \'%s\' errored on proposal from \'%s\': %s' % (TC['name'], TS['name'], e))
437                                 ProposalErrors[TC['name']] = e
438                                 continue
439                         if propose is None:
440                                 AcceptedScore += TC['weight']
441                                 self.logger.debug('Upstream \'%s\' accepted proposal' % (TC['name'],))
442                         elif propose == 'orphan':
443                                 self.logger.debug('Upstream \'%s\' considered proposal an orphan' % (TC['name'],))
444                                 ProposalErrors[TC['name']] = propose
445                         else:
446                                 RejectedScore += TC['weight']
447                                 Rejections[TC['name']] = propose
448                                 try:
449                                         propose = propose['reject-reason']
450                                 except:
451                                         pass
452                                 self.logger.error('Upstream \'%s\' rejected proposed block from \'%s\': %s' % (TC['name'], TS['name'], propose))
453                 
454                 if Rejections:
455                         RPInfo = {
456                                 'merkleTree': newMerkleTree,
457                                 'AcceptedScore': AcceptedScore,
458                                 'RejectedScore': RejectedScore,
459                                 'Rejections': Rejections,
460                                 'ProposalErrors': ProposalErrors,
461                         }
462                         self.RejectedProposal = RPInfo
463                         
464                         try:
465                                 global _filecounter
466                                 _filecounter += 1
467                                 import pickle
468                                 with open('RejectedProposals/%d_%d' % (int(time()), _filecounter), 'wb') as f:
469                                         pickle.dump(RPInfo, f)
470                         except IOError:
471                                 pass
472                 
473                 TotalScore = AcceptedScore + RejectedScore
474                 
475                 return (AcceptedScore, TotalScore)
476         
477         def _updateMerkleTree_fromTS(self, TS):
478                 MP = self._CallGBT(TS)
479                 newMerkleTree = self._ProcessGBT(MP, TS)
480                 
481                 # Some versions of bitcoinrpc ServiceProxy have problems copying/pickling, so just store name and URI for now
482                 newMerkleTree.source = TS['name']
483                 newMerkleTree.source_uri = TS['uri']
484                 
485                 (AcceptedScore, TotalScore) = self._CheckTemplate(newMerkleTree, TS)
486                 if TotalScore is None:
487                         return (0, newMerkleTree)
488                 
489                 if TotalScore:
490                         AcceptRatio = AcceptedScore / TotalScore
491                 else:
492                         AcceptRatio = 0.0
493                 
494                 self.logger.debug('Template from \'%s\' has %s acceptance ratio and score of %s' % (TS['name'], AcceptRatio, AcceptedScore))
495                 
496                 if AcceptRatio <= self.MinimumTemplateAcceptanceRatio:
497                         return None
498                 
499                 if TotalScore < self.MinimumTemplateScore:
500                         return None
501                 
502                 return (AcceptRatio, newMerkleTree)
503         
504         def _updateMerkleTree_I(self):
505                 Best = (-1, None)
506                 for TSPriList in self.TemplateSources:
507                         # FIXME: Implement weighting
508                         for i in range(len(TSPriList)):
509                                 TS = TSPriList.pop(0)
510                                 TSPriList.append(TS)
511                                 
512                                 try:
513                                         r = self._updateMerkleTree_fromTS(TS)
514                                         if r is None:
515                                                 # Failed completely
516                                                 continue
517                                         
518                                         (AcceptRatio, newMerkleTree) = r
519                                         
520                                         # NOTE: If you're going to try to remove this preference for the highest block, you need to (at least) stop _ProcessGBT from calling updateBlock whenever it sees a new high
521                                         AcceptRatio += newMerkleTree.MP['height']
522                                         
523                                         self.logger.debug('Template from \'%s\' has %s acceptance ratio at height %s' % (TS['name'], AcceptRatio, newMerkleTree.MP['height']))
524                                         if Best[0] < AcceptRatio:
525                                                 Best = (AcceptRatio, newMerkleTree)
526                                                 if AcceptRatio == 1:
527                                                         break
528                                 except:
529                                         if TSPriList == self.TemplateSources[-1] and i == len(TSPriList) - 1 and Best[1] is None:
530                                                 raise
531                                         else:
532                                                 self.logger.error(traceback.format_exc())
533                 
534                 BestMT = Best[1]
535                 if BestMT is None:
536                         raise RuntimeError('Failed to create usable template')
537                 
538                 self.logger.debug('Updating merkle tree with template from \'%s\'' % (BestMT.source,))
539                 MP = BestMT.MP
540                 blkbasics = (MP['_prevBlock'], MP['height'], MP['_bits'])
541                 if blkbasics != self.currentBlock:
542                         self.updateBlock(*blkbasics, _HBH=(MP['previousblockhash'], MP['bits']))
543                 self.currentMerkleTree = BestMT
544         
545         def _updateMerkleTree(self):
546                 global now
547                 self.logger.debug('Polling for new block template')
548                 self.nextMerkleUpdate = now + self.TxnUpdateRetryWait
549                 
550                 self._updateMerkleTree_I()
551                 
552                 self.lastMerkleUpdate = now
553                 self.nextMerkleUpdate = now + self.MinimumTxnUpdateWait
554                 
555                 if self.needMerkle == 2:
556                         self.needMerkle = 1
557                         self.needMerkleSince = now
558         
559         def updateMerkleTree(self):
560                 global now
561                 now = time()
562                 self._updateMerkleTree()
563         
564         def makeCoinbase(self, height):
565                 now = int(time())
566                 if now > _makeCoinbase[0]:
567                         _makeCoinbase[0] = now
568                         _makeCoinbase[1] = 0
569                 else:
570                         _makeCoinbase[1] += 1
571                 rv = self.CoinbasePrefix
572                 rv += pack('>L', now) + pack('>Q', _makeCoinbase[1]).lstrip(b'\0')
573                 # NOTE: Not using varlenEncode, since this is always guaranteed to be < 100
574                 rv = bytes( (len(rv),) ) + rv
575                 for v in self.CoinbaseAux.values():
576                         rv += v
577                 if len(rv) > 95:
578                         t = time()
579                         if self.overflowed < t - 300:
580                                 self.logger.warning('Overflowing coinbase data! %d bytes long' % (len(rv),))
581                                 self.overflowed = t
582                                 self.isOverflowed = True
583                         rv = rv[:95]
584                 else:
585                         self.isOverflowed = False
586                 rv = bitcoin.script.encodeUNum(height) + rv
587                 return rv
588         
589         def makeMerkleRoot(self, merkleTree, height):
590                 cbtxn = merkleTree.data[0]
591                 cb = self.makeCoinbase(height=height)
592                 cbtxn.setCoinbase(cb)
593                 cbtxn.assemble()
594                 merkleRoot = merkleTree.merkleRoot()
595                 return (merkleRoot, merkleTree, cb)
596         
597         _doing_last = None
598         def _doing(self, what):
599                 if self._doing_last == what:
600                         self._doing_i += 1
601                         return
602                 global now
603                 if self._doing_last:
604                         self.logger.debug("Switching from (%4dx in %5.3f seconds) %s => %s" % (self._doing_i, now - self._doing_s, self._doing_last, what))
605                 self._doing_last = what
606                 self._doing_i = 1
607                 self._doing_s = now
608         
609         def _floodWarning(self, now, wid, wmsgf = None, doin = True, logf = None):
610                 if doin is True:
611                         doin = self._doing_last
612                         def a(f = wmsgf):
613                                 return lambda: "%s (doing %s)" % (f(), doin)
614                         wmsgf = a()
615                 winfo = self.lastWarning.setdefault(wid, [0, None])
616                 (lastTime, lastDoing) = winfo
617                 if now <= lastTime + max(5, self.MinimumTxnUpdateWait):
618                         return
619                 winfo[0] = now
620                 nowDoing = doin
621                 winfo[1] = nowDoing
622                 if logf is None:
623                         logf = self.logger.warning
624                 logf(wmsgf() if wmsgf else doin)
625         
626         def _makeOne(self, putf, merkleTree, height):
627                 MakingAtThisHeight = self.currentBlock[1]
628                 MR = self.makeMerkleRoot(merkleTree, height=height)
629                 # Only add it if the height hasn't changed in the meantime, to avoid a race
630                 if self.currentBlock[1] == MakingAtThisHeight:
631                         putf(MR)
632         
633         def makeClear(self):
634                 self._doing('clear merkle roots')
635                 self._makeOne(self.clearMerkleRoots.put, self.curClearMerkleTree, height=self.currentBlock[1])
636         
637         def makeNext(self):
638                 self._doing('longpoll merkle roots')
639                 self._makeOne(self.nextMerkleRoots.put, self.nextMerkleTree, height=self.currentBlock[1] + 1)
640         
641         def makeRegular(self):
642                 self._doing('regular merkle roots')
643                 self._makeOne(self.merkleRoots.append, self.currentMerkleTree, height=self.currentBlock[1])
644         
645         def merkleMaker_II(self):
646                 global now
647                 
648                 # No bits = no mining :(
649                 if not self.ready:
650                         return self._updateMerkleTree()
651                 
652                 # First, ensure we have the minimum clear, next, and regular (in that order)
653                 if self.clearMerkleRoots.qsize() < self.WorkQueueSizeClear[0]:
654                         return self.makeClear()
655                 if self.nextMerkleRoots.qsize() < self.WorkQueueSizeLongpoll[0]:
656                         return self.makeNext()
657                 if len(self.merkleRoots) < self.WorkQueueSizeRegular[0]:
658                         return self.makeRegular()
659                 
660                 # If we've met the minimum requirements, consider updating the merkle tree
661                 if self.nextMerkleUpdate <= now:
662                         return self._updateMerkleTree()
663                 
664                 # Finally, fill up clear, next, and regular until we've met the maximums
665                 if self.clearMerkleRoots.qsize() < self.WorkQueueSizeClear[1]:
666                         return self.makeClear()
667                 if self.nextMerkleRoots.qsize() < self.WorkQueueSizeLongpoll[1]:
668                         return self.makeNext()
669                 if len(self.merkleRoots) < self.WorkQueueSizeRegular[1] or self.merkleRoots[0][1] != self.currentMerkleTree:
670                         return self.makeRegular()
671                 
672                 # Nothing left to do, fire onBlockUpdate event (if appropriate) and sleep
673                 if self.needMerkle == 1:
674                         self.onBlockUpdate()
675                         self.needMerkle = False
676                 self._doing('idle')
677                 # TODO: rather than sleepspin, block until MinimumTxnUpdateWait expires or threading.Condition(?)
678                 sleep(self.IdleSleepTime)
679         
680         def merkleMaker_I(self):
681                 global now
682                 now = time()
683                 
684                 self.merkleMaker_II()
685                 
686                 if self.needMerkle == 1 and now > self.needMerkleSince + self.WarningDelayTxnLongpoll:
687                         self._floodWarning(now, 'NeedMerkle', lambda: 'Transaction-longpoll requested %d seconds ago, and still not ready. Is your server fast enough to keep up with your configured WorkQueueSizeRegular maximum?' % (now - self.needMerkleSince,))
688                 if now > self.nextMerkleUpdate + self.WarningDelayMerkleUpdate:
689                         self._floodWarning(now, 'MerkleUpdate', lambda: "Haven't updated the merkle tree in at least %d seconds! Is your server fast enough to keep up with your configured work queue minimums?" % (now - self.lastMerkleUpdate,))
690         
691         def run(self):
692                 while True:
693                         try:
694                                 self.merkleMaker_I()
695                         except:
696                                 self.logger.critical(traceback.format_exc())
697         
698         def start(self, *a, **k):
699                 self._prepare()
700                 super().start(*a, **k)
701         
702         def getMRD(self):
703                 try:
704                         MRD = self.merkleRoots.pop()
705                         self.LowestMerkleRoots = min(len(self.merkleRoots), self.LowestMerkleRoots)
706                         rollPrevBlk = False
707                 except IndexError:
708                         qsz = self.clearMerkleRoots.qsize()
709                         if qsz < 0x10:
710                                 self.logger.warning('clearMerkleRoots running out! only %d left' % (qsz,))
711                         MRD = None
712                         while MRD is None:
713                                 MRD = self.clearMerkleRoots.get()
714                         self.LowestClearMerkleRoots = min(self.clearMerkleRoots.qsize(), self.LowestClearMerkleRoots)
715                         rollPrevBlk = True
716                 (merkleRoot, merkleTree, cb) = MRD
717                 (prevBlock, height, bits) = self.currentBlock
718                 return (merkleRoot, merkleTree, cb, prevBlock, bits, rollPrevBlk)
719         
720         def getMC(self, wantClear = False):
721                 if not self.ready:
722                         with self.readyCV:
723                                 while not self.ready:
724                                         self.readyCV.wait()
725                 (prevBlock, height, bits) = self.currentBlock
726                 mt = self.curClearMerkleTree if wantClear else self.currentMerkleTree
727                 cb = self.makeCoinbase(height=height)
728                 rollPrevBlk = (mt == self.curClearMerkleTree)
729                 return (height, mt, cb, prevBlock, bits, rollPrevBlk)
730
731 # merkleMaker tests
732 def _test():
733         global now
734         now = 1337039788
735         MM = merkleMaker()
736         reallogger = MM.logger
737         class fakelogger:
738                 LO = False
739                 def critical(self, *a):
740                         if self.LO > 1: return
741                         reallogger.critical(*a)
742                 def warning(self, *a):
743                         if self.LO: return
744                         reallogger.warning(*a)
745                 def debug(self, *a):
746                         pass
747         MM.logger = fakelogger()
748         class NMTClass:
749                 pass
750         
751         # _makeBlockSafe tests
752         from copy import deepcopy
753         MP = {
754                 'coinbasevalue':50,
755         }
756         txnlist = [b'\0', b'\x01', b'\x02']
757         txninfo = [{'fee':0, 'sigops':1}, {'fee':5, 'sigops':10000}, {'fee':0, 'sigops':10001}]
758         def MBS(LO = 0):
759                 m = deepcopy( (MP, txnlist, txninfo) )
760                 MM.logger.LO = LO
761                 try:
762                         MM._makeBlockSafe(*m)
763                 except:
764                         if LO < 2:
765                                 raise
766                 else:
767                         assert LO < 2  # An expected error wasn't thrown
768                 if 'POTInfo' in m[0]:
769                         del m[0]['POTInfo']
770                 return m
771         MM.POT = 0
772         assert MBS() == (MP, txnlist[:2], txninfo[:2])
773         txninfo[2]['fee'] = 1
774         MPx = deepcopy(MP)
775         MPx['coinbasevalue'] -= 1
776         assert MBS() == (MPx, txnlist[:2], txninfo[:2])
777         txninfo[2]['sigops'] = 1
778         assert MBS(1) == (MP, txnlist, txninfo)
779         # APOT tests
780         MM.POT = 2
781         txnlist.append(b'\x03')
782         txninfo.append({'fee':1, 'sigops':0})
783         MPx = deepcopy(MP)
784         MPx['coinbasevalue'] -= 1
785         assert MBS() == (MPx, txnlist[:3], txninfo[:3])
786         # POT tests
787         MM.POT = 1
788         MM.Greedy = True
789         txninfo[1]['fee'] = 0
790         txninfo[2]['fee'] = 0
791         assert MBS(1) == (MP, txnlist, txninfo)
792         # _ProcessGBT tests
793         def makeCoinbaseTxn(coinbaseValue, useCoinbaser = True, prevBlockHex = None):
794                 txn = Txn.new()
795                 txn.addOutput(coinbaseValue, b'')
796                 return txn
797         MM.makeCoinbaseTxn = makeCoinbaseTxn
798         MM.updateBlock = lambda *a, **ka: None
799         gbt = {
800                 'transactions': [
801                         {'data': '11', 'depends': [], 'fee': 1, 'sigops': 1},
802                         {'data': '11', 'depends': [], 'fee': 0, 'sigops': 1},
803                         {'data': '11', 'depends': [], 'fee': 0, 'sigops': 1},
804                         {'data': '11', 'depends': [], 'fee': 1, 'sigops': 2}
805                 ],
806                 'height': 219507,
807                 'coinbasevalue': 3,
808                 'previousblockhash': '000000000000012806bc100006dc83220bd9c2ac2709dc14a0d0fa1d6f9b733c',
809                 'version': 1,
810                 'bits': '1a05a6b1'
811         }
812         nMT = MM._ProcessGBT(gbt)
813         assert len(nMT.data) == 5
814         nMT.data[0].disassemble()
815         assert sum(outp[0] for outp in nMT.data[0].outputs) == 3
816         MM.POT = 2
817         nMT = MM._ProcessGBT(gbt)
818         assert len(nMT.data) in (2, 4)
819         nMT.data[0].disassemble()
820         assert sum(outp[0] for outp in nMT.data[0].outputs) == 2
821
822 _test()