Bugfix: Fix SubsidyAlgo
[bitcoin:eloipool.git] / merklemaker.py
1 # Eloipool - Python Bitcoin pool server
2 # Copyright (C) 2011-2013  Luke Dashjr <luke-jr+eloipool@utopios.org>
3 #
4 # This program is free software: you can redistribute it and/or modify
5 # it under the terms of the GNU Affero General Public License as
6 # published by the Free Software Foundation, either version 3 of the
7 # License, or (at your option) any later version.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12 # GNU Affero General Public License for more details.
13 #
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
16
17 from binascii import b2a_hex
18 import bitcoin.script
19 from bitcoin.script import countSigOps
20 from bitcoin.txn import Txn
21 from bitcoin.varlen import varlenEncode, varlenDecode
22 from collections import deque
23 from copy import deepcopy
24 from queue import Queue
25 import jsonrpc
26 import logging
27 from math import log
28 from merkletree import MerkleTree
29 import socket
30 from struct import pack
31 import threading
32 from time import sleep, time
33 import traceback
34
35 _makeCoinbase = [0, 0]
36 _filecounter = 0
37
38 def MakeBlockHeader(MRD):
39         (merkleRoot, merkleTree, coinbase, prevBlock, bits) = MRD[:5]
40         timestamp = pack('<L', int(time()))
41         hdr = b'\2\0\0\0' + prevBlock + merkleRoot + timestamp + bits + b'iolE'
42         return hdr
43
44 def assembleBlock(blkhdr, txlist):
45         payload = blkhdr
46         payload += varlenEncode(len(txlist))
47         for tx in txlist:
48                 payload += tx.data
49         return payload
50
51 class merkleMaker(threading.Thread):
52         GBTCaps = [
53                 'coinbasevalue',
54                 'coinbase/append',
55                 'coinbase',
56                 'generation',
57                 'time',
58                 'transactions/remove',
59                 'prevblock',
60         ]
61         GBTReq = {
62                 'capabilities': GBTCaps,
63         }
64         GMPReq = {
65                 'capabilities': GBTCaps,
66                 'tx': 'obj',
67         }
68         
69         def __init__(self, *a, **k):
70                 super().__init__(*a, **k)
71                 self.daemon = True
72                 self.logger = logging.getLogger('merkleMaker')
73                 self.CoinbasePrefix = b''
74                 self.CoinbaseAux = {}
75                 self.isOverflowed = False
76                 self.lastWarning = {}
77                 self.MinimumTxnUpdateWait = 5
78                 self.overflowed = 0
79                 self.DifficultyChangeMod = 2016
80                 self.MinimumTemplateAcceptanceRatio = 0
81                 self.MinimumTemplateScore = 1
82                 self.currentBlock = (None, None, None)
83                 self.lastBlock = (None, None, None)
84                 self.SubsidyAlgo = lambda height: 5000000000 >> (height // 210000)
85         
86         def _prepare(self):
87                 self.TemplateSources = list(getattr(self, 'TemplateSources', ()))
88                 self.TemplateChecks = list(getattr(self, 'TemplateChecks', ()))
89                 if getattr(self, 'BlockSubmissions', None) is None:
90                         self.BlockSubmissions = ()
91                 if hasattr(self, 'UpstreamURI'):
92                         self.TemplateSources.append({
93                                 'name': 'UpstreamURI',
94                                 'uri': self.UpstreamURI,
95                         })
96                 URI2Name = {}
97                 Name2URI = {}
98                 for a in (self.TemplateSources + self.TemplateChecks + list(self.BlockSubmissions)):
99                         if not ('name' in a and 'uri' in a):
100                                 continue
101                         URI2Name.setdefault(a['uri'], a['name'])
102                         Name2URI.setdefault(a['name'], a['uri'])
103                 def URINamePair(a, defname):
104                         if 'name' not in a:
105                                 a['name'] = URI2Name.get(a['uri'], defname)
106                         elif 'uri' not in a:
107                                 a['uri'] = Name2URI[a['name']]
108                 _URI2Access = {}
109                 def URI2Access(uri):
110                         if uri not in _URI2Access:
111                                 access = jsonrpc.ServiceProxy(uri)
112                                 access.OldGMP = False
113                                 _URI2Access[uri] = access
114                         return _URI2Access[uri]
115                 LeveledTS = {}
116                 for i in range(len(self.TemplateSources)):
117                         TS = self.TemplateSources[i]
118                         URINamePair(TS, 'TemplateSources[%u]' % (i,))
119                         TS.setdefault('priority', 0)
120                         TS.setdefault('weight', 1)
121                         TS['access'] = URI2Access(TS['uri'])
122                         LeveledTS.setdefault(TS['priority'], []).append(TS)
123                 LeveledTS = tuple(x[1] for x in sorted(LeveledTS.items()))
124                 self.TemplateSources = LeveledTS
125                 for i in range(len(self.TemplateChecks)):
126                         TC = self.TemplateChecks[i]
127                         URINamePair(TC, 'TemplateChecks[%u]' % (i,))
128                         TC.setdefault('unanimous', False)
129                         TC.setdefault('weight', 1)
130                         TC['access'] = URI2Access(TC['uri'])
131                 for i in range(len(getattr(self, 'BlockSubmissions', ()))):
132                         BS = self.BlockSubmissions[i]
133                         URINamePair(BS, 'BlockSubmissions[%u]' % (i,))
134                         BS['access'] = URI2Access(BS['uri'])
135                 
136                 self.ready = False
137                 self.readyCV = threading.Condition()
138                 
139                 self.currentMerkleTree = None
140                 self.merkleRoots = deque(maxlen=self.WorkQueueSizeRegular[1])
141                 self.LowestMerkleRoots = self.WorkQueueSizeRegular[1]
142                 
143                 if not hasattr(self, 'WorkQueueSizeClear'):
144                         self.WorkQueueSizeClear = self.WorkQueueSizeLongpoll
145                 self._MaxClearSize = max(self.WorkQueueSizeClear[1], self.WorkQueueSizeLongpoll[1])
146                 self.clearMerkleRoots = Queue(self._MaxClearSize)
147                 self.LowestClearMerkleRoots = self.WorkQueueSizeClear[1]
148                 self.nextMerkleRoots = Queue(self._MaxClearSize)
149                 
150                 if not hasattr(self, 'WarningDelay'):
151                         self.WarningDelay = max(15, self.MinimumTxnUpdateWait * 2)
152                 if not hasattr(self, 'WarningDelayTxnLongpoll'):
153                         self.WarningDelayTxnLongpoll = self.WarningDelay
154                 if not hasattr(self, 'WarningDelayMerkleUpdate'):
155                         self.WarningDelayMerkleUpdate = self.WarningDelay
156                 
157                 self.lastMerkleUpdate = 0
158                 self.nextMerkleUpdate = 0
159         
160         def createClearMerkleTree(self, height):
161                 subsidy = self.SubsidyAlgo(height)
162                 cbtxn = self.makeCoinbaseTxn(subsidy, False)
163                 cbtxn.assemble()
164                 return MerkleTree([cbtxn])
165         
166         def updateBlock(self, newBlock, height = None, bits = None, _HBH = None):
167                 if newBlock == self.currentBlock[0]:
168                         if height in (None, self.currentBlock[1]) and bits in (None, self.currentBlock[2]):
169                                 return
170                         if not self.currentBlock[2] is None:
171                                 self.logger.error('Was working on block with wrong specs: %s (height: %d->%d; bits: %s->%s' % (
172                                         b2a_hex(newBlock[::-1]).decode('utf8'),
173                                         self.currentBlock[1],
174                                         height,
175                                         b2a_hex(self.currentBlock[2][::-1]).decode('utf8'),
176                                         b2a_hex(bits[::-1]).decode('utf8'),
177                                 ))
178                                 if self.needMerkle == 1:
179                                         self.needMerkle = False
180                                 self.onBlockUpdate()
181                 
182                 # Old block is invalid
183                 if self.currentBlock[0] != newBlock:
184                         self.lastBlock = self.currentBlock
185                 
186                 lastHeight = self.currentBlock[1]
187                 if height is None:
188                         height = self.currentBlock[1] + 1
189                 if bits is None:
190                         if height % self.DifficultyChangeMod == 1 or self.currentBlock[2] is None:
191                                 self.logger.warning('New block: %s (height %d; bits: UNKNOWN)' % (b2a_hex(newBlock[::-1]).decode('utf8'), height))
192                                 # Pretend to be 1 lower height, so we possibly retain nextMerkleRoots
193                                 self.currentBlock = (None, height - 1, None)
194                                 OCMR = self.clearMerkleRoots
195                                 self.clearMerkleRoots = Queue(self.WorkQueueSizeClear[1])
196                                 if OCMR.empty():
197                                         OCMR.put(None)
198                                 self.merkleRoots.clear()
199                                 self.ready = False
200                                 return
201                         else:
202                                 bits = self.currentBlock[2]
203                 
204                 if _HBH is None:
205                         _HBH = (b2a_hex(newBlock[::-1]).decode('utf8'), b2a_hex(bits[::-1]).decode('utf8'))
206                 self.logger.info('New block: %s (height: %d; bits: %s)' % (_HBH[0], height, _HBH[1]))
207                 self.currentBlock = (newBlock, height, bits)
208                 
209                 if lastHeight != height:
210                         # TODO: Perhaps reuse clear merkle trees more intelligently
211                         OCMR = self.clearMerkleRoots
212                         if lastHeight == height - 1:
213                                 self.curClearMerkleTree = self.nextMerkleTree
214                                 self.clearMerkleRoots = self.nextMerkleRoots
215                                 self.logger.debug('Adopting next-height clear merkleroots :)')
216                         else:
217                                 if lastHeight:
218                                         self.logger.warning('Change from height %d->%d; no longpoll merkleroots available!' % (lastHeight, height))
219                                 self.curClearMerkleTree = self.createClearMerkleTree(height)
220                                 self.clearMerkleRoots = Queue(self.WorkQueueSizeClear[1])
221                         if OCMR.empty():
222                                 OCMR.put(None)
223                         self.nextMerkleTree = self.createClearMerkleTree(height + 1)
224                         self.nextMerkleRoots = Queue(self._MaxClearSize)
225                 else:
226                         self.logger.debug('Already using clear merkleroots for this height')
227                 self.currentMerkleTree = self.curClearMerkleTree
228                 self.merkleRoots.clear()
229                 
230                 if not self.ready:
231                         self.ready = True
232                         with self.readyCV:
233                                 self.readyCV.notify_all()
234                 
235                 self.needMerkle = 2
236                 self.onBlockChange()
237         
238         def _trimBlock(self, MP, txnlist, txninfo, floodn, msgf):
239                 fee = txninfo[-1].get('fee', None)
240                 if fee is None:
241                         raise self._floodCritical(now, floodn, doin=msgf('fees unknown'))
242                 if fee:
243                         # FIXME: coinbasevalue is *not* guaranteed to exist here
244                         MP['coinbasevalue'] -= fee
245                 
246                 txnlist[-1:] = ()
247                 txninfo[-1:] = ()
248                 
249                 return True
250         
251         # Aggressive "Power Of Two": Remove transactions even with fees to reach our goal
252         def _APOT(self, txninfopot, MP, POTInfo):
253                 feeTxnsTrimmed = 0
254                 feesTrimmed = 0
255                 for txn in txninfopot:
256                         if txn.get('fee') is None:
257                                 self._floodWarning(now, 'APOT-No-Fees', doin='Upstream didn\'t provide fee information required for aggressive POT', logf=self.logger.info)
258                                 return
259                         if not txn['fee']:
260                                 continue
261                         feesTrimmed += txn['fee']
262                         feeTxnsTrimmed += 1
263                 MP['coinbasevalue'] -= feesTrimmed
264                 
265                 POTInfo[2] = [feeTxnsTrimmed, feesTrimmed]
266                 self._floodWarning(now, 'POT-Trimming-Fees', doin='Aggressive POT trimming %d transactions with %d.%08d BTC total fees' % (feeTxnsTrimmed, feesTrimmed//100000000, feesTrimmed % 100000000), logf=self.logger.debug)
267                 
268                 return True
269         
270         def _makeBlockSafe(self, MP, txnlist, txninfo):
271                 blocksize = sum(map(len, txnlist)) + 80
272                 while blocksize > 934464:  # 1 "MB" limit - 64 KB breathing room
273                         txnsize = len(txnlist[-1])
274                         self._trimBlock(MP, txnlist, txninfo, 'SizeLimit', lambda x: 'Making blocks over 1 MB size limit (%d bytes; %s)' % (blocksize, x))
275                         blocksize -= txnsize
276                 
277                 # NOTE: This check doesn't work at all without BIP22 transaction obj format
278                 blocksigops = sum(a.get('sigops', 0) for a in txninfo)
279                 while blocksigops > 19488:  # 20k limit - 0x200 breathing room
280                         txnsigops = txninfo[-1]['sigops']
281                         self._trimBlock(MP, txnlist, txninfo, 'SigOpLimit', lambda x: 'Making blocks over 20k SigOp limit (%d; %s)' % (blocksigops, x))
282                         blocksigops -= txnsigops
283                 
284                 # Aim to produce blocks with "Power Of Two" transaction counts
285                 # This helps avoid any chance of someone abusing CVE-2012-2459 with them
286                 POTMode = getattr(self, 'POT', 1)
287                 txncount = len(txnlist) + 1
288                 if POTMode:
289                         feetxncount = txncount
290                         for i in range(txncount - 2, -1, -1):
291                                 if 'fee' not in txninfo[i] or txninfo[i]['fee']:
292                                         break
293                                 feetxncount -= 1
294                         
295                         if getattr(self, 'Greedy', None):
296                                 # Aim to cut off extra zero-fee transactions on the end
297                                 # NOTE: not cutting out ones intermixed, in case of dependencies
298                                 idealtxncount = feetxncount
299                         else:
300                                 idealtxncount = txncount
301                         
302                         pot = 2**int(log(idealtxncount, 2))
303                         POTInfo = MP['POTInfo'] = [[idealtxncount, feetxncount, txncount], [pot, None], None]
304                         if pot < idealtxncount:
305                                 if pot * 2 <= txncount:
306                                         pot *= 2
307                                 elif pot >= feetxncount:
308                                         pass
309                                 elif POTMode > 1:
310                                         if self._APOT(txninfo[pot-1:], MP, POTInfo):
311                                                 # Trimmed even transactions with fees
312                                                 pass
313                                         else:
314                                                 pot = idealtxncount
315                                                 self._floodWarning(now, 'Non-POT', doin='Making merkle tree with %d transactions (ideal: %d; max: %d)' % (pot, idealtxncount, txncount))
316                                 else:
317                                         pot = idealtxncount
318                         POTInfo[1][1] = pot
319                         pot -= 1
320                         txnlist[pot:] = ()
321                         txninfo[pot:] = ()
322         
323         def _CallGBT(self, TS):
324                 access = TS['access']
325                 self.logger.debug('Requesting new template from \'%s\'' % (TS['name'],))
326                 try:
327                         # First, try BIP 22 standard getblocktemplate :)
328                         MP = access.getblocktemplate(self.GBTReq)
329                         access.OldGMP = False
330                 except:
331                         try:
332                                 # Failing that, give BIP 22 draft (2012-02 through 2012-07) getmemorypool a chance
333                                 MP = access.getmemorypool(self.GMPReq)
334                         except:
335                                 try:
336                                         # Finally, fall back to bitcoind 0.5/0.6 getmemorypool
337                                         MP = access.getmemorypool()
338                                 except:
339                                         MP = False
340                         if MP is False:
341                                 # This way, we get the error from the BIP22 call if the old one fails too
342                                 raise
343                         
344                         # Pre-BIP22 server (bitcoind <0.7 or Eloipool <20120513)
345                         if not access.OldGMP:
346                                 access.OldGMP = True
347                                 self.logger.warning('Upstream \'%s\' is not BIP 22 compatible' % (TS['name'],))
348                 
349                 return MP
350         
351         def _ProcessGBT(self, MP, TS = None):
352                 oMP = MP
353                 MP = deepcopy(MP)
354                 
355                 prevBlock = bytes.fromhex(MP['previousblockhash'])[::-1]
356                 if 'height' not in MP:
357                         MP['height'] = TS['access'].getinfo()['blocks'] + 1
358                 height = MP['height']
359                 bits = bytes.fromhex(MP['bits'])[::-1]
360                 (MP['_bits'], MP['_prevBlock']) = (bits, prevBlock)
361                 if (prevBlock, height, bits) != self.currentBlock and (self.currentBlock[1] is None or height > self.currentBlock[1]):
362                         self.updateBlock(prevBlock, height, bits, _HBH=(MP['previousblockhash'], MP['bits']))
363                 
364                 txnlist = MP['transactions']
365                 if len(txnlist) and isinstance(txnlist[0], dict):
366                         txninfo = txnlist
367                         txnlist = tuple(a['data'] for a in txnlist)
368                 elif 'transactionfees' in MP:
369                         # Backward compatibility with pre-BIP22 gmp_fees branch
370                         txninfo = [{'fee':a} for a in MP['transactionfees']]
371                 else:
372                         # Backward compatibility with pre-BIP22 hex-only (bitcoind <0.7, Eloipool <future)
373                         txninfo = [{}] * len(txnlist)
374                 # TODO: cache Txn or at least txid from previous merkle roots?
375                 txnlist = [a for a in map(bytes.fromhex, txnlist)]
376                 
377                 self._makeBlockSafe(MP, txnlist, txninfo)
378                 
379                 cbtxn = self.makeCoinbaseTxn(MP['coinbasevalue'], prevBlockHex = MP['previousblockhash'])
380                 cbtxn.setCoinbase(b'\0\0')
381                 cbtxn.assemble()
382                 txnlist.insert(0, cbtxn.data)
383                 txninfo.insert(0, {
384                 })
385                 
386                 txnlist = [a for a in map(Txn, txnlist[1:])]
387                 txnlist.insert(0, cbtxn)
388                 txnlist = list(txnlist)
389                 newMerkleTree = MerkleTree(txnlist)
390                 newMerkleTree.POTInfo = MP.get('POTInfo')
391                 newMerkleTree.MP = MP
392                 newMerkleTree.oMP = oMP
393                 
394                 return newMerkleTree
395         
396         def _CheckTemplate(self, newMerkleTree, TS):
397                 TCList = self.TemplateChecks
398                 if not TCList:
399                         if 'proposal' not in newMerkleTree.oMP.get('capabilities', ()):
400                                 return (None, None)
401                         TCList = (
402                                 {
403                                         'name': TS['name'],
404                                         'access': TS['access'],
405                                         'unanimous': True,
406                                         'weight': 1,
407                                 },
408                         )
409                 
410                 MP = newMerkleTree.MP
411                 (prevBlock, height, bits) = (MP['_prevBlock'], MP['height'], MP['_bits'])
412                 txnlist = newMerkleTree.data
413                 cbtxn = txnlist[0]
414                 
415                 coinbase = self.makeCoinbase(height=height)
416                 cbtxn.setCoinbase(coinbase)
417                 cbtxn.assemble()
418                 merkleRoot = newMerkleTree.merkleRoot()
419                 MRD = (merkleRoot, newMerkleTree, coinbase, prevBlock, bits)
420                 blkhdr = MakeBlockHeader(MRD)
421                 data = assembleBlock(blkhdr, txnlist)
422                 ProposeReq = {
423                         "mode": "proposal",
424                         "data": b2a_hex(data).decode('utf8'),
425                 }
426                 
427                 AcceptedScore = 0
428                 RejectedScore = 0
429                 Rejections = {}
430                 ProposalErrors = {}
431                 for TC in TCList:
432                         caccess = TC['access']
433                         try:
434                                 propose = caccess.getblocktemplate(ProposeReq)
435                         except (socket.error, ValueError) as e:
436                                 self.logger.error('Upstream \'%s\' errored on proposal from \'%s\': %s' % (TC['name'], TS['name'], e))
437                                 ProposalErrors[TC['name']] = e
438                                 continue
439                         if propose is None:
440                                 AcceptedScore += TC['weight']
441                                 self.logger.debug('Upstream \'%s\' accepted proposal' % (TC['name'],))
442                         elif propose == 'orphan':
443                                 self.logger.debug('Upstream \'%s\' considered proposal an orphan' % (TC['name'],))
444                                 ProposalErrors[TC['name']] = propose
445                         else:
446                                 RejectedScore += TC['weight']
447                                 Rejections[TC['name']] = propose
448                                 try:
449                                         propose = propose['reject-reason']
450                                 except:
451                                         pass
452                                 self.logger.error('Upstream \'%s\' rejected proposed block from \'%s\': %s' % (TC['name'], TS['name'], propose))
453                 
454                 if Rejections:
455                         RPInfo = {
456                                 'merkleTree': newMerkleTree,
457                                 'AcceptedScore': AcceptedScore,
458                                 'RejectedScore': RejectedScore,
459                                 'Rejections': Rejections,
460                                 'ProposalErrors': ProposalErrors,
461                         }
462                         self.RejectedProposal = RPInfo
463                         
464                         try:
465                                 global _filecounter
466                                 _filecounter += 1
467                                 import pickle
468                                 with open('RejectedProposals/%d_%d' % (int(time()), _filecounter), 'wb') as f:
469                                         pickle.dump(RPInfo, f)
470                         except IOError:
471                                 pass
472                 
473                 TotalScore = AcceptedScore + RejectedScore
474                 
475                 return (AcceptedScore, TotalScore)
476         
477         def _updateMerkleTree_fromTS(self, TS):
478                 MP = self._CallGBT(TS)
479                 newMerkleTree = self._ProcessGBT(MP, TS)
480                 
481                 # Some versions of bitcoinrpc ServiceProxy have problems copying/pickling, so just store name and URI for now
482                 newMerkleTree.source = TS['name']
483                 newMerkleTree.source_uri = TS['uri']
484                 
485                 (AcceptedScore, TotalScore) = self._CheckTemplate(newMerkleTree, TS)
486                 if TotalScore is None:
487                         return (0, newMerkleTree)
488                 
489                 if TotalScore:
490                         AcceptRatio = AcceptedScore / TotalScore
491                 else:
492                         AcceptRatio = 0.0
493                 
494                 self.logger.debug('Template from \'%s\' has %s acceptance ratio and score of %s' % (TS['name'], AcceptRatio, AcceptedScore))
495                 
496                 if AcceptRatio <= self.MinimumTemplateAcceptanceRatio:
497                         return None
498                 
499                 if TotalScore < self.MinimumTemplateScore:
500                         return None
501                 
502                 return (AcceptRatio, newMerkleTree)
503         
504         def _updateMerkleTree_I(self):
505                 Best = (-1, None)
506                 for TSPriList in self.TemplateSources:
507                         # FIXME: Implement weighting
508                         for i in range(len(TSPriList)):
509                                 TS = TSPriList.pop(0)
510                                 TSPriList.append(TS)
511                                 
512                                 try:
513                                         r = self._updateMerkleTree_fromTS(TS)
514                                         if r is None:
515                                                 # Failed completely
516                                                 continue
517                                         
518                                         (AcceptRatio, newMerkleTree) = r
519                                         
520                                         # NOTE: If you're going to try to remove this preference for the highest block, you need to (at least) stop _ProcessGBT from calling updateBlock whenever it sees a new high
521                                         AcceptRatio += newMerkleTree.MP['height']
522                                         
523                                         if Best[0] < AcceptRatio:
524                                                 Best = r
525                                                 if AcceptRatio == 1:
526                                                         break
527                                 except:
528                                         if TSPriList == self.TemplateSources[-1] and i == len(TSPriList) - 1 and Best[1] is None:
529                                                 raise
530                                         else:
531                                                 self.logger.error(traceback.format_exc())
532                 
533                 BestMT = Best[1]
534                 if BestMT is None:
535                         raise RuntimeError('Failed to create usable template')
536                 
537                 self.logger.debug('Updating merkle tree with template from \'%s\'' % (BestMT.source,))
538                 MP = BestMT.MP
539                 blkbasics = (MP['_prevBlock'], MP['height'], MP['_bits'])
540                 if blkbasics != self.currentBlock:
541                         self.updateBlock(*blkbasics, _HBH=(MP['previousblockhash'], MP['bits']))
542                 self.currentMerkleTree = BestMT
543         
544         def _updateMerkleTree(self):
545                 global now
546                 self.logger.debug('Polling for new block template')
547                 self.nextMerkleUpdate = now + self.TxnUpdateRetryWait
548                 
549                 self._updateMerkleTree_I()
550                 
551                 self.lastMerkleUpdate = now
552                 self.nextMerkleUpdate = now + self.MinimumTxnUpdateWait
553                 
554                 if self.needMerkle == 2:
555                         self.needMerkle = 1
556                         self.needMerkleSince = now
557         
558         def updateMerkleTree(self):
559                 global now
560                 now = time()
561                 self._updateMerkleTree()
562         
563         def makeCoinbase(self, height):
564                 now = int(time())
565                 if now > _makeCoinbase[0]:
566                         _makeCoinbase[0] = now
567                         _makeCoinbase[1] = 0
568                 else:
569                         _makeCoinbase[1] += 1
570                 rv = self.CoinbasePrefix
571                 rv += pack('>L', now) + pack('>Q', _makeCoinbase[1]).lstrip(b'\0')
572                 # NOTE: Not using varlenEncode, since this is always guaranteed to be < 100
573                 rv = bytes( (len(rv),) ) + rv
574                 for v in self.CoinbaseAux.values():
575                         rv += v
576                 if len(rv) > 95:
577                         t = time()
578                         if self.overflowed < t - 300:
579                                 self.logger.warning('Overflowing coinbase data! %d bytes long' % (len(rv),))
580                                 self.overflowed = t
581                                 self.isOverflowed = True
582                         rv = rv[:95]
583                 else:
584                         self.isOverflowed = False
585                 rv = bitcoin.script.encodeUNum(height) + rv
586                 return rv
587         
588         def makeMerkleRoot(self, merkleTree, height):
589                 cbtxn = merkleTree.data[0]
590                 cb = self.makeCoinbase(height=height)
591                 cbtxn.setCoinbase(cb)
592                 cbtxn.assemble()
593                 merkleRoot = merkleTree.merkleRoot()
594                 return (merkleRoot, merkleTree, cb)
595         
596         _doing_last = None
597         def _doing(self, what):
598                 if self._doing_last == what:
599                         self._doing_i += 1
600                         return
601                 global now
602                 if self._doing_last:
603                         self.logger.debug("Switching from (%4dx in %5.3f seconds) %s => %s" % (self._doing_i, now - self._doing_s, self._doing_last, what))
604                 self._doing_last = what
605                 self._doing_i = 1
606                 self._doing_s = now
607         
608         def _floodWarning(self, now, wid, wmsgf = None, doin = True, logf = None):
609                 if doin is True:
610                         doin = self._doing_last
611                         def a(f = wmsgf):
612                                 return lambda: "%s (doing %s)" % (f(), doin)
613                         wmsgf = a()
614                 winfo = self.lastWarning.setdefault(wid, [0, None])
615                 (lastTime, lastDoing) = winfo
616                 if now <= lastTime + max(5, self.MinimumTxnUpdateWait):
617                         return
618                 winfo[0] = now
619                 nowDoing = doin
620                 winfo[1] = nowDoing
621                 if logf is None:
622                         logf = self.logger.warning
623                 logf(wmsgf() if wmsgf else doin)
624         
625         def _makeOne(self, putf, merkleTree, height):
626                 MakingAtThisHeight = self.currentBlock[1]
627                 MR = self.makeMerkleRoot(merkleTree, height=height)
628                 # Only add it if the height hasn't changed in the meantime, to avoid a race
629                 if self.currentBlock[1] == MakingAtThisHeight:
630                         putf(MR)
631         
632         def makeClear(self):
633                 self._doing('clear merkle roots')
634                 self._makeOne(self.clearMerkleRoots.put, self.curClearMerkleTree, height=self.currentBlock[1])
635         
636         def makeNext(self):
637                 self._doing('longpoll merkle roots')
638                 self._makeOne(self.nextMerkleRoots.put, self.nextMerkleTree, height=self.currentBlock[1] + 1)
639         
640         def makeRegular(self):
641                 self._doing('regular merkle roots')
642                 self._makeOne(self.merkleRoots.append, self.currentMerkleTree, height=self.currentBlock[1])
643         
644         def merkleMaker_II(self):
645                 global now
646                 
647                 # No bits = no mining :(
648                 if not self.ready:
649                         return self._updateMerkleTree()
650                 
651                 # First, ensure we have the minimum clear, next, and regular (in that order)
652                 if self.clearMerkleRoots.qsize() < self.WorkQueueSizeClear[0]:
653                         return self.makeClear()
654                 if self.nextMerkleRoots.qsize() < self.WorkQueueSizeLongpoll[0]:
655                         return self.makeNext()
656                 if len(self.merkleRoots) < self.WorkQueueSizeRegular[0]:
657                         return self.makeRegular()
658                 
659                 # If we've met the minimum requirements, consider updating the merkle tree
660                 if self.nextMerkleUpdate <= now:
661                         return self._updateMerkleTree()
662                 
663                 # Finally, fill up clear, next, and regular until we've met the maximums
664                 if self.clearMerkleRoots.qsize() < self.WorkQueueSizeClear[1]:
665                         return self.makeClear()
666                 if self.nextMerkleRoots.qsize() < self.WorkQueueSizeLongpoll[1]:
667                         return self.makeNext()
668                 if len(self.merkleRoots) < self.WorkQueueSizeRegular[1] or self.merkleRoots[0][1] != self.currentMerkleTree:
669                         return self.makeRegular()
670                 
671                 # Nothing left to do, fire onBlockUpdate event (if appropriate) and sleep
672                 if self.needMerkle == 1:
673                         self.onBlockUpdate()
674                         self.needMerkle = False
675                 self._doing('idle')
676                 # TODO: rather than sleepspin, block until MinimumTxnUpdateWait expires or threading.Condition(?)
677                 sleep(self.IdleSleepTime)
678         
679         def merkleMaker_I(self):
680                 global now
681                 now = time()
682                 
683                 self.merkleMaker_II()
684                 
685                 if self.needMerkle == 1 and now > self.needMerkleSince + self.WarningDelayTxnLongpoll:
686                         self._floodWarning(now, 'NeedMerkle', lambda: 'Transaction-longpoll requested %d seconds ago, and still not ready. Is your server fast enough to keep up with your configured WorkQueueSizeRegular maximum?' % (now - self.needMerkleSince,))
687                 if now > self.nextMerkleUpdate + self.WarningDelayMerkleUpdate:
688                         self._floodWarning(now, 'MerkleUpdate', lambda: "Haven't updated the merkle tree in at least %d seconds! Is your server fast enough to keep up with your configured work queue minimums?" % (now - self.lastMerkleUpdate,))
689         
690         def run(self):
691                 while True:
692                         try:
693                                 self.merkleMaker_I()
694                         except:
695                                 self.logger.critical(traceback.format_exc())
696         
697         def start(self, *a, **k):
698                 self._prepare()
699                 super().start(*a, **k)
700         
701         def getMRD(self):
702                 try:
703                         MRD = self.merkleRoots.pop()
704                         self.LowestMerkleRoots = min(len(self.merkleRoots), self.LowestMerkleRoots)
705                         rollPrevBlk = False
706                 except IndexError:
707                         qsz = self.clearMerkleRoots.qsize()
708                         if qsz < 0x10:
709                                 self.logger.warning('clearMerkleRoots running out! only %d left' % (qsz,))
710                         MRD = None
711                         while MRD is None:
712                                 MRD = self.clearMerkleRoots.get()
713                         self.LowestClearMerkleRoots = min(self.clearMerkleRoots.qsize(), self.LowestClearMerkleRoots)
714                         rollPrevBlk = True
715                 (merkleRoot, merkleTree, cb) = MRD
716                 (prevBlock, height, bits) = self.currentBlock
717                 return (merkleRoot, merkleTree, cb, prevBlock, bits, rollPrevBlk)
718         
719         def getMC(self, wantClear = False):
720                 if not self.ready:
721                         with self.readyCV:
722                                 while not self.ready:
723                                         self.readyCV.wait()
724                 (prevBlock, height, bits) = self.currentBlock
725                 mt = self.curClearMerkleTree if wantClear else self.currentMerkleTree
726                 cb = self.makeCoinbase(height=height)
727                 rollPrevBlk = (mt == self.curClearMerkleTree)
728                 return (height, mt, cb, prevBlock, bits, rollPrevBlk)
729
730 # merkleMaker tests
731 def _test():
732         global now
733         now = 1337039788
734         MM = merkleMaker()
735         reallogger = MM.logger
736         class fakelogger:
737                 LO = False
738                 def critical(self, *a):
739                         if self.LO > 1: return
740                         reallogger.critical(*a)
741                 def warning(self, *a):
742                         if self.LO: return
743                         reallogger.warning(*a)
744                 def debug(self, *a):
745                         pass
746         MM.logger = fakelogger()
747         class NMTClass:
748                 pass
749         
750         # _makeBlockSafe tests
751         from copy import deepcopy
752         MP = {
753                 'coinbasevalue':50,
754         }
755         txnlist = [b'\0', b'\x01', b'\x02']
756         txninfo = [{'fee':0, 'sigops':1}, {'fee':5, 'sigops':10000}, {'fee':0, 'sigops':10001}]
757         def MBS(LO = 0):
758                 m = deepcopy( (MP, txnlist, txninfo) )
759                 MM.logger.LO = LO
760                 try:
761                         MM._makeBlockSafe(*m)
762                 except:
763                         if LO < 2:
764                                 raise
765                 else:
766                         assert LO < 2  # An expected error wasn't thrown
767                 if 'POTInfo' in m[0]:
768                         del m[0]['POTInfo']
769                 return m
770         MM.POT = 0
771         assert MBS() == (MP, txnlist[:2], txninfo[:2])
772         txninfo[2]['fee'] = 1
773         MPx = deepcopy(MP)
774         MPx['coinbasevalue'] -= 1
775         assert MBS() == (MPx, txnlist[:2], txninfo[:2])
776         txninfo[2]['sigops'] = 1
777         assert MBS(1) == (MP, txnlist, txninfo)
778         # APOT tests
779         MM.POT = 2
780         txnlist.append(b'\x03')
781         txninfo.append({'fee':1, 'sigops':0})
782         MPx = deepcopy(MP)
783         MPx['coinbasevalue'] -= 1
784         assert MBS() == (MPx, txnlist[:3], txninfo[:3])
785         # POT tests
786         MM.POT = 1
787         MM.Greedy = True
788         txninfo[1]['fee'] = 0
789         txninfo[2]['fee'] = 0
790         assert MBS(1) == (MP, txnlist, txninfo)
791         # _ProcessGBT tests
792         def makeCoinbaseTxn(coinbaseValue, useCoinbaser = True, prevBlockHex = None):
793                 txn = Txn.new()
794                 txn.addOutput(coinbaseValue, b'')
795                 return txn
796         MM.makeCoinbaseTxn = makeCoinbaseTxn
797         MM.updateBlock = lambda *a, **ka: None
798         gbt = {
799                 'transactions': [
800                         {'data': '11', 'depends': [], 'fee': 1, 'sigops': 1},
801                         {'data': '11', 'depends': [], 'fee': 0, 'sigops': 1},
802                         {'data': '11', 'depends': [], 'fee': 0, 'sigops': 1},
803                         {'data': '11', 'depends': [], 'fee': 1, 'sigops': 2}
804                 ],
805                 'height': 219507,
806                 'coinbasevalue': 3,
807                 'previousblockhash': '000000000000012806bc100006dc83220bd9c2ac2709dc14a0d0fa1d6f9b733c',
808                 'version': 1,
809                 'bits': '1a05a6b1'
810         }
811         nMT = MM._ProcessGBT(gbt)
812         assert len(nMT.data) == 5
813         nMT.data[0].disassemble()
814         assert sum(outp[0] for outp in nMT.data[0].outputs) == 3
815         MM.POT = 2
816         nMT = MM._ProcessGBT(gbt)
817         assert len(nMT.data) in (2, 4)
818         nMT.data[0].disassemble()
819         assert sum(outp[0] for outp in nMT.data[0].outputs) == 2
820
821 _test()