Merge branch 'multisource'
[bitcoin:eloipool.git] / merklemaker.py
1 # Eloipool - Python Bitcoin pool server
2 # Copyright (C) 2011-2013  Luke Dashjr <luke-jr+eloipool@utopios.org>
3 #
4 # This program is free software: you can redistribute it and/or modify
5 # it under the terms of the GNU Affero General Public License as
6 # published by the Free Software Foundation, either version 3 of the
7 # License, or (at your option) any later version.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12 # GNU Affero General Public License for more details.
13 #
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
16
17 from binascii import b2a_hex
18 import bitcoin.script
19 from bitcoin.script import countSigOps
20 from bitcoin.txn import Txn
21 from bitcoin.varlen import varlenEncode, varlenDecode
22 from collections import deque
23 from copy import deepcopy
24 from queue import Queue
25 import jsonrpc
26 import logging
27 from math import log
28 from merkletree import MerkleTree
29 import socket
30 from struct import pack
31 import threading
32 from time import sleep, time
33 import traceback
34
35 _makeCoinbase = [0, 0]
36 _filecounter = 0
37
38 def MakeBlockHeader(MRD):
39         (merkleRoot, merkleTree, coinbase, prevBlock, bits) = MRD[:5]
40         timestamp = pack('<L', int(time()))
41         hdr = b'\2\0\0\0' + prevBlock + merkleRoot + timestamp + bits + b'iolE'
42         return hdr
43
44 def assembleBlock(blkhdr, txlist):
45         payload = blkhdr
46         payload += varlenEncode(len(txlist))
47         for tx in txlist:
48                 payload += tx.data
49         return payload
50
51 class merkleMaker(threading.Thread):
52         GBTCaps = [
53                 'coinbasevalue',
54                 'coinbase/append',
55                 'coinbase',
56                 'generation',
57                 'time',
58                 'transactions/remove',
59                 'prevblock',
60         ]
61         GBTReq = {
62                 'capabilities': GBTCaps,
63         }
64         GMPReq = {
65                 'capabilities': GBTCaps,
66                 'tx': 'obj',
67         }
68         
69         def __init__(self, *a, **k):
70                 super().__init__(*a, **k)
71                 self.daemon = True
72                 self.logger = logging.getLogger('merkleMaker')
73                 self.CoinbasePrefix = b''
74                 self.CoinbaseAux = {}
75                 self.isOverflowed = False
76                 self.lastWarning = {}
77                 self.MinimumTxnUpdateWait = 5
78                 self.overflowed = 0
79                 self.DifficultyChangeMod = 2016
80                 self.MinimumTemplateAcceptanceRatio = 0
81                 self.MinimumTemplateScore = 1
82                 self.currentBlock = (None, None, None)
83                 self.lastBlock = (None, None, None)
84         
85         def _prepare(self):
86                 self.TemplateSources = list(getattr(self, 'TemplateSources', ()))
87                 self.TemplateChecks = list(getattr(self, 'TemplateChecks', ()))
88                 if getattr(self, 'BlockSubmissions', None) is None:
89                         self.BlockSubmissions = ()
90                 if hasattr(self, 'UpstreamURI'):
91                         self.TemplateSources.append({
92                                 'name': 'UpstreamURI',
93                                 'uri': self.UpstreamURI,
94                         })
95                 URI2Name = {}
96                 Name2URI = {}
97                 for a in (self.TemplateSources + self.TemplateChecks + list(self.BlockSubmissions)):
98                         if not ('name' in a and 'uri' in a):
99                                 continue
100                         URI2Name.setdefault(a['uri'], a['name'])
101                         Name2URI.setdefault(a['name'], a['uri'])
102                 def URINamePair(a, defname):
103                         if 'name' not in a:
104                                 a['name'] = URI2Name.get(a['uri'], defname)
105                         elif 'uri' not in a:
106                                 a['uri'] = Name2URI[a['name']]
107                 _URI2Access = {}
108                 def URI2Access(uri):
109                         if uri not in _URI2Access:
110                                 access = jsonrpc.ServiceProxy(uri)
111                                 access.OldGMP = False
112                                 _URI2Access[uri] = access
113                         return _URI2Access[uri]
114                 LeveledTS = {}
115                 for i in range(len(self.TemplateSources)):
116                         TS = self.TemplateSources[i]
117                         URINamePair(TS, 'TemplateSources[%u]' % (i,))
118                         TS.setdefault('priority', 0)
119                         TS.setdefault('weight', 1)
120                         TS['access'] = URI2Access(TS['uri'])
121                         LeveledTS.setdefault(TS['priority'], []).append(TS)
122                 LeveledTS = tuple(x[1] for x in sorted(LeveledTS.items()))
123                 self.TemplateSources = LeveledTS
124                 for i in range(len(self.TemplateChecks)):
125                         TC = self.TemplateChecks[i]
126                         URINamePair(TC, 'TemplateChecks[%u]' % (i,))
127                         TC.setdefault('unanimous', False)
128                         TC.setdefault('weight', 1)
129                         TC['access'] = URI2Access(TC['uri'])
130                 for i in range(len(getattr(self, 'BlockSubmissions', ()))):
131                         BS = self.BlockSubmissions[i]
132                         URINamePair(BS, 'BlockSubmissions[%u]' % (i,))
133                         BS['access'] = URI2Access(BS['uri'])
134                 
135                 self.ready = False
136                 self.readyCV = threading.Condition()
137                 
138                 self.currentMerkleTree = None
139                 self.merkleRoots = deque(maxlen=self.WorkQueueSizeRegular[1])
140                 self.LowestMerkleRoots = self.WorkQueueSizeRegular[1]
141                 
142                 if not hasattr(self, 'WorkQueueSizeClear'):
143                         self.WorkQueueSizeClear = self.WorkQueueSizeLongpoll
144                 self._MaxClearSize = max(self.WorkQueueSizeClear[1], self.WorkQueueSizeLongpoll[1])
145                 self.clearMerkleRoots = Queue(self._MaxClearSize)
146                 self.LowestClearMerkleRoots = self.WorkQueueSizeClear[1]
147                 self.nextMerkleRoots = Queue(self._MaxClearSize)
148                 
149                 if not hasattr(self, 'WarningDelay'):
150                         self.WarningDelay = max(15, self.MinimumTxnUpdateWait * 2)
151                 if not hasattr(self, 'WarningDelayTxnLongpoll'):
152                         self.WarningDelayTxnLongpoll = self.WarningDelay
153                 if not hasattr(self, 'WarningDelayMerkleUpdate'):
154                         self.WarningDelayMerkleUpdate = self.WarningDelay
155                 
156                 self.lastMerkleUpdate = 0
157                 self.nextMerkleUpdate = 0
158         
159         def createClearMerkleTree(self, height):
160                 subsidy = 5000000000 >> (height // 210000)
161                 cbtxn = self.makeCoinbaseTxn(subsidy, False)
162                 cbtxn.assemble()
163                 return MerkleTree([cbtxn])
164         
165         def updateBlock(self, newBlock, height = None, bits = None, _HBH = None):
166                 if newBlock == self.currentBlock[0]:
167                         if height in (None, self.currentBlock[1]) and bits in (None, self.currentBlock[2]):
168                                 return
169                         if not self.currentBlock[2] is None:
170                                 self.logger.error('Was working on block with wrong specs: %s (height: %d->%d; bits: %s->%s' % (
171                                         b2a_hex(newBlock[::-1]).decode('utf8'),
172                                         self.currentBlock[1],
173                                         height,
174                                         b2a_hex(self.currentBlock[2][::-1]).decode('utf8'),
175                                         b2a_hex(bits[::-1]).decode('utf8'),
176                                 ))
177                                 if self.needMerkle == 1:
178                                         self.needMerkle = False
179                                 self.onBlockUpdate()
180                 
181                 # Old block is invalid
182                 if self.currentBlock[0] != newBlock:
183                         self.lastBlock = self.currentBlock
184                 
185                 lastHeight = self.currentBlock[1]
186                 if height is None:
187                         height = self.currentBlock[1] + 1
188                 if bits is None:
189                         if height % self.DifficultyChangeMod == 1 or self.currentBlock[2] is None:
190                                 self.logger.warning('New block: %s (height %d; bits: UNKNOWN)' % (b2a_hex(newBlock[::-1]).decode('utf8'), height))
191                                 # Pretend to be 1 lower height, so we possibly retain nextMerkleRoots
192                                 self.currentBlock = (None, height - 1, None)
193                                 OCMR = self.clearMerkleRoots
194                                 self.clearMerkleRoots = Queue(self.WorkQueueSizeClear[1])
195                                 if OCMR.empty():
196                                         OCMR.put(None)
197                                 self.merkleRoots.clear()
198                                 self.ready = False
199                                 return
200                         else:
201                                 bits = self.currentBlock[2]
202                 
203                 if _HBH is None:
204                         _HBH = (b2a_hex(newBlock[::-1]).decode('utf8'), b2a_hex(bits[::-1]).decode('utf8'))
205                 self.logger.info('New block: %s (height: %d; bits: %s)' % (_HBH[0], height, _HBH[1]))
206                 self.currentBlock = (newBlock, height, bits)
207                 
208                 if lastHeight != height:
209                         # TODO: Perhaps reuse clear merkle trees more intelligently
210                         OCMR = self.clearMerkleRoots
211                         if lastHeight == height - 1:
212                                 self.curClearMerkleTree = self.nextMerkleTree
213                                 self.clearMerkleRoots = self.nextMerkleRoots
214                                 self.logger.debug('Adopting next-height clear merkleroots :)')
215                         else:
216                                 if lastHeight:
217                                         self.logger.warning('Change from height %d->%d; no longpoll merkleroots available!' % (lastHeight, height))
218                                 self.curClearMerkleTree = self.createClearMerkleTree(height)
219                                 self.clearMerkleRoots = Queue(self.WorkQueueSizeClear[1])
220                         if OCMR.empty():
221                                 OCMR.put(None)
222                         self.nextMerkleTree = self.createClearMerkleTree(height + 1)
223                         self.nextMerkleRoots = Queue(self._MaxClearSize)
224                 else:
225                         self.logger.debug('Already using clear merkleroots for this height')
226                 self.currentMerkleTree = self.curClearMerkleTree
227                 self.merkleRoots.clear()
228                 
229                 if not self.ready:
230                         self.ready = True
231                         with self.readyCV:
232                                 self.readyCV.notify_all()
233                 
234                 self.needMerkle = 2
235                 self.onBlockChange()
236         
237         def _trimBlock(self, MP, txnlist, txninfo, floodn, msgf):
238                 fee = txninfo[-1].get('fee', None)
239                 if fee is None:
240                         raise self._floodCritical(now, floodn, doin=msgf('fees unknown'))
241                 if fee:
242                         # FIXME: coinbasevalue is *not* guaranteed to exist here
243                         MP['coinbasevalue'] -= fee
244                 
245                 txnlist[-1:] = ()
246                 txninfo[-1:] = ()
247                 
248                 return True
249         
250         # Aggressive "Power Of Two": Remove transactions even with fees to reach our goal
251         def _APOT(self, txninfopot, MP, POTInfo):
252                 feeTxnsTrimmed = 0
253                 feesTrimmed = 0
254                 for txn in txninfopot:
255                         if txn.get('fee') is None:
256                                 self._floodWarning(now, 'APOT-No-Fees', doin='Upstream didn\'t provide fee information required for aggressive POT', logf=self.logger.info)
257                                 return
258                         if not txn['fee']:
259                                 continue
260                         feesTrimmed += txn['fee']
261                         feeTxnsTrimmed += 1
262                 MP['coinbasevalue'] -= feesTrimmed
263                 
264                 POTInfo[2] = [feeTxnsTrimmed, feesTrimmed]
265                 self._floodWarning(now, 'POT-Trimming-Fees', doin='Aggressive POT trimming %d transactions with %d.%08d BTC total fees' % (feeTxnsTrimmed, feesTrimmed//100000000, feesTrimmed % 100000000), logf=self.logger.debug)
266                 
267                 return True
268         
269         def _makeBlockSafe(self, MP, txnlist, txninfo):
270                 blocksize = sum(map(len, txnlist)) + 80
271                 while blocksize > 934464:  # 1 "MB" limit - 64 KB breathing room
272                         txnsize = len(txnlist[-1])
273                         self._trimBlock(MP, txnlist, txninfo, 'SizeLimit', lambda x: 'Making blocks over 1 MB size limit (%d bytes; %s)' % (blocksize, x))
274                         blocksize -= txnsize
275                 
276                 # NOTE: This check doesn't work at all without BIP22 transaction obj format
277                 blocksigops = sum(a.get('sigops', 0) for a in txninfo)
278                 while blocksigops > 19488:  # 20k limit - 0x200 breathing room
279                         txnsigops = txninfo[-1]['sigops']
280                         self._trimBlock(MP, txnlist, txninfo, 'SigOpLimit', lambda x: 'Making blocks over 20k SigOp limit (%d; %s)' % (blocksigops, x))
281                         blocksigops -= txnsigops
282                 
283                 # Aim to produce blocks with "Power Of Two" transaction counts
284                 # This helps avoid any chance of someone abusing CVE-2012-2459 with them
285                 POTMode = getattr(self, 'POT', 1)
286                 txncount = len(txnlist) + 1
287                 if POTMode:
288                         feetxncount = txncount
289                         for i in range(txncount - 2, -1, -1):
290                                 if 'fee' not in txninfo[i] or txninfo[i]['fee']:
291                                         break
292                                 feetxncount -= 1
293                         
294                         if getattr(self, 'Greedy', None):
295                                 # Aim to cut off extra zero-fee transactions on the end
296                                 # NOTE: not cutting out ones intermixed, in case of dependencies
297                                 idealtxncount = feetxncount
298                         else:
299                                 idealtxncount = txncount
300                         
301                         pot = 2**int(log(idealtxncount, 2))
302                         POTInfo = MP['POTInfo'] = [[idealtxncount, feetxncount, txncount], [pot, None], None]
303                         if pot < idealtxncount:
304                                 if pot * 2 <= txncount:
305                                         pot *= 2
306                                 elif pot >= feetxncount:
307                                         pass
308                                 elif POTMode > 1 and self._APOT(txninfo[pot-1:], MP, POTInfo):
309                                         # Trimmed even transactions with fees
310                                         pass
311                                 else:
312                                         pot = idealtxncount
313                                         self._floodWarning(now, 'Non-POT', doin='Making merkle tree with %d transactions (ideal: %d; max: %d)' % (pot, idealtxncount, txncount))
314                         POTInfo[1][1] = pot
315                         pot -= 1
316                         txnlist[pot:] = ()
317                         txninfo[pot:] = ()
318         
319         def _CallGBT(self, TS):
320                 access = TS['access']
321                 self.logger.debug('Requesting new template from \'%s\'' % (TS['name'],))
322                 try:
323                         # First, try BIP 22 standard getblocktemplate :)
324                         MP = access.getblocktemplate(self.GBTReq)
325                         access.OldGMP = False
326                 except:
327                         try:
328                                 # Failing that, give BIP 22 draft (2012-02 through 2012-07) getmemorypool a chance
329                                 MP = access.getmemorypool(self.GMPReq)
330                         except:
331                                 try:
332                                         # Finally, fall back to bitcoind 0.5/0.6 getmemorypool
333                                         MP = access.getmemorypool()
334                                 except:
335                                         MP = False
336                         if MP is False:
337                                 # This way, we get the error from the BIP22 call if the old one fails too
338                                 raise
339                         
340                         # Pre-BIP22 server (bitcoind <0.7 or Eloipool <20120513)
341                         if not access.OldGMP:
342                                 access.OldGMP = True
343                                 self.logger.warning('Upstream \'%s\' is not BIP 22 compatible' % (TS['name'],))
344                 
345                 return MP
346         
347         def _ProcessGBT(self, MP, TS = None):
348                 oMP = MP
349                 MP = deepcopy(MP)
350                 
351                 prevBlock = bytes.fromhex(MP['previousblockhash'])[::-1]
352                 if 'height' not in MP:
353                         MP['height'] = TS['access'].getinfo()['blocks'] + 1
354                 height = MP['height']
355                 bits = bytes.fromhex(MP['bits'])[::-1]
356                 (MP['_bits'], MP['_prevBlock']) = (bits, prevBlock)
357                 if (prevBlock, height, bits) != self.currentBlock and (self.currentBlock[1] is None or height > self.currentBlock[1]):
358                         self.updateBlock(prevBlock, height, bits, _HBH=(MP['previousblockhash'], MP['bits']))
359                 
360                 txnlist = MP['transactions']
361                 if len(txnlist) and isinstance(txnlist[0], dict):
362                         txninfo = txnlist
363                         txnlist = tuple(a['data'] for a in txnlist)
364                 elif 'transactionfees' in MP:
365                         # Backward compatibility with pre-BIP22 gmp_fees branch
366                         txninfo = [{'fee':a} for a in MP['transactionfees']]
367                 else:
368                         # Backward compatibility with pre-BIP22 hex-only (bitcoind <0.7, Eloipool <future)
369                         txninfo = [{}] * len(txnlist)
370                 # TODO: cache Txn or at least txid from previous merkle roots?
371                 txnlist = [a for a in map(bytes.fromhex, txnlist)]
372                 
373                 self._makeBlockSafe(MP, txnlist, txninfo)
374                 
375                 cbtxn = self.makeCoinbaseTxn(MP['coinbasevalue'])
376                 cbtxn.setCoinbase(b'\0\0')
377                 cbtxn.assemble()
378                 txnlist.insert(0, cbtxn.data)
379                 txninfo.insert(0, {
380                 })
381                 
382                 txnlist = [a for a in map(Txn, txnlist[1:])]
383                 txnlist.insert(0, cbtxn)
384                 txnlist = list(txnlist)
385                 newMerkleTree = MerkleTree(txnlist)
386                 newMerkleTree.POTInfo = MP.get('POTInfo')
387                 newMerkleTree.MP = MP
388                 newMerkleTree.oMP = oMP
389                 
390                 return newMerkleTree
391         
392         def _CheckTemplate(self, newMerkleTree, TS):
393                 TCList = self.TemplateChecks
394                 if not TCList:
395                         if 'proposal' not in newMerkleTree.oMP.get('capabilities', ()):
396                                 return (None, None)
397                         TCList = (
398                                 {
399                                         'name': TS['name'],
400                                         'access': TS['access'],
401                                         'unanimous': True,
402                                         'weight': 1,
403                                 },
404                         )
405                 
406                 MP = newMerkleTree.MP
407                 (prevBlock, height, bits) = (MP['_prevBlock'], MP['height'], MP['_bits'])
408                 txnlist = newMerkleTree.data
409                 cbtxn = txnlist[0]
410                 
411                 coinbase = self.makeCoinbase(height=height)
412                 cbtxn.setCoinbase(coinbase)
413                 cbtxn.assemble()
414                 merkleRoot = newMerkleTree.merkleRoot()
415                 MRD = (merkleRoot, newMerkleTree, coinbase, prevBlock, bits)
416                 blkhdr = MakeBlockHeader(MRD)
417                 data = assembleBlock(blkhdr, txnlist)
418                 ProposeReq = {
419                         "mode": "proposal",
420                         "data": b2a_hex(data).decode('utf8'),
421                 }
422                 
423                 AcceptedScore = 0
424                 RejectedScore = 0
425                 Rejections = {}
426                 ProposalErrors = {}
427                 for TC in TCList:
428                         caccess = TC['access']
429                         try:
430                                 propose = caccess.getblocktemplate(ProposeReq)
431                         except (socket.error, ValueError) as e:
432                                 self.logger.error('Upstream \'%s\' errored on proposal from \'%s\': %s' % (TC['name'], TS['name'], e))
433                                 ProposalErrors[TC['name']] = e
434                                 continue
435                         if propose is None:
436                                 AcceptedScore += TC['weight']
437                                 self.logger.debug('Upstream \'%s\' accepted proposal' % (TC['name'],))
438                         elif propose == 'orphan':
439                                 self.logger.debug('Upstream \'%s\' considered proposal an orphan' % (TC['name'],))
440                                 ProposalErrors[TC['name']] = propose
441                         else:
442                                 RejectedScore += TC['weight']
443                                 Rejections[TC['name']] = propose
444                                 try:
445                                         propose = propose['reject-reason']
446                                 except:
447                                         pass
448                                 self.logger.error('Upstream \'%s\' rejected proposed block from \'%s\': %s' % (TC['name'], TS['name'], propose))
449                 
450                 if Rejections:
451                         RPInfo = {
452                                 'merkleTree': newMerkleTree,
453                                 'AcceptedScore': AcceptedScore,
454                                 'RejectedScore': RejectedScore,
455                                 'Rejections': Rejections,
456                                 'ProposalErrors': ProposalErrors,
457                         }
458                         self.RejectedProposal = RPInfo
459                         
460                         try:
461                                 global _filecounter
462                                 _filecounter += 1
463                                 import pickle
464                                 with open('RejectedProposals/%d_%d' % (int(time()), _filecounter), 'wb') as f:
465                                         pickle.dump(RPInfo, f)
466                         except IOError:
467                                 pass
468                 
469                 TotalScore = AcceptedScore + RejectedScore
470                 
471                 return (AcceptedScore, TotalScore)
472         
473         def _updateMerkleTree_fromTS(self, TS):
474                 MP = self._CallGBT(TS)
475                 newMerkleTree = self._ProcessGBT(MP, TS)
476                 
477                 # Some versions of bitcoinrpc ServiceProxy have problems copying/pickling, so just store name and URI for now
478                 newMerkleTree.source = TS['name']
479                 newMerkleTree.source_uri = TS['uri']
480                 
481                 (AcceptedScore, TotalScore) = self._CheckTemplate(newMerkleTree, TS)
482                 if TotalScore is None:
483                         return (0, newMerkleTree)
484                 
485                 AcceptRatio = AcceptedScore / TotalScore
486                 
487                 self.logger.debug('Template from \'%s\' has %s acceptance ratio and score of %s' % (TS['name'], AcceptRatio, AcceptedScore))
488                 
489                 if AcceptRatio <= self.MinimumTemplateAcceptanceRatio:
490                         return None
491                 
492                 if TotalScore < self.MinimumTemplateScore:
493                         return None
494                 
495                 return (AcceptRatio, newMerkleTree)
496         
497         def _updateMerkleTree_I(self):
498                 Best = (-1, None)
499                 for TSPriList in self.TemplateSources:
500                         # FIXME: Implement weighting
501                         for i in range(len(TSPriList)):
502                                 TS = TSPriList.pop(0)
503                                 TSPriList.append(TS)
504                                 
505                                 try:
506                                         r = self._updateMerkleTree_fromTS(TS)
507                                         if r is None:
508                                                 # Failed completely
509                                                 continue
510                                         
511                                         (AcceptRatio, newMerkleTree) = r
512                                         
513                                         # NOTE: If you're going to try to remove this preference for the highest block, you need to (at least) stop _ProcessGBT from calling updateBlock whenever it sees a new high
514                                         AcceptRatio += newMerkleTree.MP['height']
515                                         
516                                         if Best[0] < AcceptRatio:
517                                                 Best = r
518                                                 if AcceptRatio == 1:
519                                                         break
520                                 except:
521                                         if TSPriList == self.TemplateSources[-1] and i == len(TSPriList) - 1 and Best[1] is None:
522                                                 raise
523                                         else:
524                                                 self.logger.error(traceback.format_exc())
525                 
526                 BestMT = Best[1]
527                 if BestMT is None:
528                         raise RuntimeError('Failed to create usable template')
529                 
530                 self.logger.debug('Updating merkle tree with template from \'%s\'' % (BestMT.source,))
531                 MP = BestMT.MP
532                 blkbasics = (MP['_prevBlock'], MP['height'], MP['_bits'])
533                 if blkbasics != self.currentBlock:
534                         self.updateBlock(*blkbasics, _HBH=(MP['previousblockhash'], MP['bits']))
535                 self.currentMerkleTree = BestMT
536         
537         def _updateMerkleTree(self):
538                 global now
539                 self.logger.debug('Polling for new block template')
540                 self.nextMerkleUpdate = now + self.TxnUpdateRetryWait
541                 
542                 self._updateMerkleTree_I()
543                 
544                 self.lastMerkleUpdate = now
545                 self.nextMerkleUpdate = now + self.MinimumTxnUpdateWait
546                 
547                 if self.needMerkle == 2:
548                         self.needMerkle = 1
549                         self.needMerkleSince = now
550         
551         def updateMerkleTree(self):
552                 global now
553                 now = time()
554                 self._updateMerkleTree()
555         
556         def makeCoinbase(self, height):
557                 now = int(time())
558                 if now > _makeCoinbase[0]:
559                         _makeCoinbase[0] = now
560                         _makeCoinbase[1] = 0
561                 else:
562                         _makeCoinbase[1] += 1
563                 rv = self.CoinbasePrefix
564                 rv += pack('>L', now) + pack('>Q', _makeCoinbase[1]).lstrip(b'\0')
565                 # NOTE: Not using varlenEncode, since this is always guaranteed to be < 100
566                 rv = bytes( (len(rv),) ) + rv
567                 for v in self.CoinbaseAux.values():
568                         rv += v
569                 if len(rv) > 95:
570                         t = time()
571                         if self.overflowed < t - 300:
572                                 self.logger.warning('Overflowing coinbase data! %d bytes long' % (len(rv),))
573                                 self.overflowed = t
574                                 self.isOverflowed = True
575                         rv = rv[:95]
576                 else:
577                         self.isOverflowed = False
578                 rv = bitcoin.script.encodeUNum(height) + rv
579                 return rv
580         
581         def makeMerkleRoot(self, merkleTree, height):
582                 cbtxn = merkleTree.data[0]
583                 cb = self.makeCoinbase(height=height)
584                 cbtxn.setCoinbase(cb)
585                 cbtxn.assemble()
586                 merkleRoot = merkleTree.merkleRoot()
587                 return (merkleRoot, merkleTree, cb)
588         
589         _doing_last = None
590         def _doing(self, what):
591                 if self._doing_last == what:
592                         self._doing_i += 1
593                         return
594                 global now
595                 if self._doing_last:
596                         self.logger.debug("Switching from (%4dx in %5.3f seconds) %s => %s" % (self._doing_i, now - self._doing_s, self._doing_last, what))
597                 self._doing_last = what
598                 self._doing_i = 1
599                 self._doing_s = now
600         
601         def _floodWarning(self, now, wid, wmsgf = None, doin = True, logf = None):
602                 if doin is True:
603                         doin = self._doing_last
604                         def a(f = wmsgf):
605                                 return lambda: "%s (doing %s)" % (f(), doin)
606                         wmsgf = a()
607                 winfo = self.lastWarning.setdefault(wid, [0, None])
608                 (lastTime, lastDoing) = winfo
609                 if now <= lastTime + max(5, self.MinimumTxnUpdateWait):
610                         return
611                 winfo[0] = now
612                 nowDoing = doin
613                 winfo[1] = nowDoing
614                 if logf is None:
615                         logf = self.logger.warning
616                 logf(wmsgf() if wmsgf else doin)
617         
618         def _makeOne(self, putf, merkleTree, height):
619                 MakingAtThisHeight = self.currentBlock[1]
620                 MR = self.makeMerkleRoot(merkleTree, height=height)
621                 # Only add it if the height hasn't changed in the meantime, to avoid a race
622                 if self.currentBlock[1] == MakingAtThisHeight:
623                         putf(MR)
624         
625         def makeClear(self):
626                 self._doing('clear merkle roots')
627                 self._makeOne(self.clearMerkleRoots.put, self.curClearMerkleTree, height=self.currentBlock[1])
628         
629         def makeNext(self):
630                 self._doing('longpoll merkle roots')
631                 self._makeOne(self.nextMerkleRoots.put, self.nextMerkleTree, height=self.currentBlock[1] + 1)
632         
633         def makeRegular(self):
634                 self._doing('regular merkle roots')
635                 self._makeOne(self.merkleRoots.append, self.currentMerkleTree, height=self.currentBlock[1])
636         
637         def merkleMaker_II(self):
638                 global now
639                 
640                 # No bits = no mining :(
641                 if not self.ready:
642                         return self._updateMerkleTree()
643                 
644                 # First, ensure we have the minimum clear, next, and regular (in that order)
645                 if self.clearMerkleRoots.qsize() < self.WorkQueueSizeClear[0]:
646                         return self.makeClear()
647                 if self.nextMerkleRoots.qsize() < self.WorkQueueSizeLongpoll[0]:
648                         return self.makeNext()
649                 if len(self.merkleRoots) < self.WorkQueueSizeRegular[0]:
650                         return self.makeRegular()
651                 
652                 # If we've met the minimum requirements, consider updating the merkle tree
653                 if self.nextMerkleUpdate <= now:
654                         return self._updateMerkleTree()
655                 
656                 # Finally, fill up clear, next, and regular until we've met the maximums
657                 if self.clearMerkleRoots.qsize() < self.WorkQueueSizeClear[1]:
658                         return self.makeClear()
659                 if self.nextMerkleRoots.qsize() < self.WorkQueueSizeLongpoll[1]:
660                         return self.makeNext()
661                 if len(self.merkleRoots) < self.WorkQueueSizeRegular[1] or self.merkleRoots[0][1] != self.currentMerkleTree:
662                         return self.makeRegular()
663                 
664                 # Nothing left to do, fire onBlockUpdate event (if appropriate) and sleep
665                 if self.needMerkle == 1:
666                         self.onBlockUpdate()
667                         self.needMerkle = False
668                 self._doing('idle')
669                 # TODO: rather than sleepspin, block until MinimumTxnUpdateWait expires or threading.Condition(?)
670                 sleep(self.IdleSleepTime)
671         
672         def merkleMaker_I(self):
673                 global now
674                 now = time()
675                 
676                 self.merkleMaker_II()
677                 
678                 if self.needMerkle == 1 and now > self.needMerkleSince + self.WarningDelayTxnLongpoll:
679                         self._floodWarning(now, 'NeedMerkle', lambda: 'Transaction-longpoll requested %d seconds ago, and still not ready. Is your server fast enough to keep up with your configured WorkQueueSizeRegular maximum?' % (now - self.needMerkleSince,))
680                 if now > self.nextMerkleUpdate + self.WarningDelayMerkleUpdate:
681                         self._floodWarning(now, 'MerkleUpdate', lambda: "Haven't updated the merkle tree in at least %d seconds! Is your server fast enough to keep up with your configured work queue minimums?" % (now - self.lastMerkleUpdate,))
682         
683         def run(self):
684                 while True:
685                         try:
686                                 self.merkleMaker_I()
687                         except:
688                                 self.logger.critical(traceback.format_exc())
689         
690         def start(self, *a, **k):
691                 self._prepare()
692                 super().start(*a, **k)
693         
694         def getMRD(self):
695                 try:
696                         MRD = self.merkleRoots.pop()
697                         self.LowestMerkleRoots = min(len(self.merkleRoots), self.LowestMerkleRoots)
698                         rollPrevBlk = False
699                 except IndexError:
700                         qsz = self.clearMerkleRoots.qsize()
701                         if qsz < 0x10:
702                                 self.logger.warning('clearMerkleRoots running out! only %d left' % (qsz,))
703                         MRD = None
704                         while MRD is None:
705                                 MRD = self.clearMerkleRoots.get()
706                         self.LowestClearMerkleRoots = min(self.clearMerkleRoots.qsize(), self.LowestClearMerkleRoots)
707                         rollPrevBlk = True
708                 (merkleRoot, merkleTree, cb) = MRD
709                 (prevBlock, height, bits) = self.currentBlock
710                 return (merkleRoot, merkleTree, cb, prevBlock, bits, rollPrevBlk)
711         
712         def getMC(self, wantClear = False):
713                 if not self.ready:
714                         with self.readyCV:
715                                 while not self.ready:
716                                         self.readyCV.wait()
717                 (prevBlock, height, bits) = self.currentBlock
718                 mt = self.curClearMerkleTree if wantClear else self.currentMerkleTree
719                 cb = self.makeCoinbase(height=height)
720                 rollPrevBlk = (mt == self.curClearMerkleTree)
721                 return (height, mt, cb, prevBlock, bits, rollPrevBlk)
722
723 # merkleMaker tests
724 def _test():
725         global now
726         now = 1337039788
727         MM = merkleMaker()
728         reallogger = MM.logger
729         class fakelogger:
730                 LO = False
731                 def critical(self, *a):
732                         if self.LO > 1: return
733                         reallogger.critical(*a)
734                 def warning(self, *a):
735                         if self.LO: return
736                         reallogger.warning(*a)
737                 def debug(self, *a):
738                         pass
739         MM.logger = fakelogger()
740         class NMTClass:
741                 pass
742         
743         # _makeBlockSafe tests
744         from copy import deepcopy
745         MP = {
746                 'coinbasevalue':50,
747         }
748         txnlist = [b'\0', b'\x01', b'\x02']
749         txninfo = [{'fee':0, 'sigops':1}, {'fee':5, 'sigops':10000}, {'fee':0, 'sigops':10001}]
750         def MBS(LO = 0):
751                 m = deepcopy( (MP, txnlist, txninfo) )
752                 MM.logger.LO = LO
753                 try:
754                         MM._makeBlockSafe(*m)
755                 except:
756                         if LO < 2:
757                                 raise
758                 else:
759                         assert LO < 2  # An expected error wasn't thrown
760                 if 'POTInfo' in m[0]:
761                         del m[0]['POTInfo']
762                 return m
763         MM.POT = 0
764         assert MBS() == (MP, txnlist[:2], txninfo[:2])
765         txninfo[2]['fee'] = 1
766         MPx = deepcopy(MP)
767         MPx['coinbasevalue'] -= 1
768         assert MBS() == (MPx, txnlist[:2], txninfo[:2])
769         txninfo[2]['sigops'] = 1
770         assert MBS(1) == (MP, txnlist, txninfo)
771         # APOT tests
772         MM.POT = 2
773         txnlist.append(b'\x03')
774         txninfo.append({'fee':1, 'sigops':0})
775         MPx = deepcopy(MP)
776         MPx['coinbasevalue'] -= 1
777         assert MBS() == (MPx, txnlist[:3], txninfo[:3])
778         # POT tests
779         MM.POT = 1
780         MM.Greedy = True
781         txninfo[1]['fee'] = 0
782         txninfo[2]['fee'] = 0
783         assert MBS(1) == (MP, txnlist, txninfo)
784         # _ProcessGBT tests
785         def makeCoinbaseTxn(coinbaseValue, useCoinbaser = True):
786                 txn = Txn.new()
787                 txn.addOutput(coinbaseValue, b'')
788                 return txn
789         MM.makeCoinbaseTxn = makeCoinbaseTxn
790         MM.updateBlock = lambda *a, **ka: None
791         gbt = {
792                 'transactions': [
793                         {'data': '11', 'depends': [], 'fee': 1, 'sigops': 1},
794                         {'data': '11', 'depends': [], 'fee': 0, 'sigops': 1},
795                         {'data': '11', 'depends': [], 'fee': 0, 'sigops': 1},
796                         {'data': '11', 'depends': [], 'fee': 1, 'sigops': 2}
797                 ],
798                 'height': 219507,
799                 'coinbasevalue': 3,
800                 'previousblockhash': '000000000000012806bc100006dc83220bd9c2ac2709dc14a0d0fa1d6f9b733c',
801                 'version': 1,
802                 'bits': '1a05a6b1'
803         }
804         nMT = MM._ProcessGBT(gbt)
805         assert len(nMT.data) == 5
806         nMT.data[0].disassemble()
807         assert sum(outp[0] for outp in nMT.data[0].outputs) == 3
808         MM.POT = 2
809         nMT = MM._ProcessGBT(gbt)
810         assert len(nMT.data) in (2, 4)
811         nMT.data[0].disassemble()
812         assert sum(outp[0] for outp in nMT.data[0].outputs) == 2
813
814 _test()