merklemaker.py

   1 # Eloipool - Python Bitcoin pool server
   2 # Copyright (C) 2011-2013  Luke Dashjr <luke-jr+eloipool@utopios.org>
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU Affero General Public License as
   6 # published by the Free Software Foundation, either version 3 of the
   7 # License, or (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU Affero General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU Affero General Public License
  15 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 from binascii import b2a_hex
  18 import bitcoin.script
  19 from bitcoin.script import countSigOps
  20 from bitcoin.txn import Txn
  21 from bitcoin.varlen import varlenEncode, varlenDecode
  22 from collections import deque
  23 from copy import deepcopy
  24 from queue import Queue
  25 import jsonrpc
  26 import logging
  27 from math import log
  28 from merkletree import MerkleTree
  29 import socket
  30 from struct import pack
  31 import threading
  32 from time import sleep, time
  33 import traceback
  34
  35 _makeCoinbase = [0, 0]
  36 _filecounter = 0
  37
  38 def MakeBlockHeader(MRD):
  39         (merkleRoot, merkleTree, coinbase, prevBlock, bits) = MRD[:5]
  40         timestamp = pack('<L', int(time()))
  41         hdr = b'\2\0\0\0' + prevBlock + merkleRoot + timestamp + bits + b'iolE'
  42         return hdr
  43
  44 def assembleBlock(blkhdr, txlist):
  45         payload = blkhdr
  46         payload += varlenEncode(len(txlist))
  47         for tx in txlist:
  48                 payload += tx.data
  49         return payload
  50
  51 class merkleMaker(threading.Thread):
  52         GBTCaps = [
  53                 'coinbasevalue',
  54                 'coinbase/append',
  55                 'coinbase',
  56                 'generation',
  57                 'time',
  58                 'transactions/remove',
  59                 'prevblock',
  60         ]
  61         GBTReq = {
  62                 'capabilities': GBTCaps,
  63         }
  64         GMPReq = {
  65                 'capabilities': GBTCaps,
  66                 'tx': 'obj',
  67         }
  68
  69         def __init__(self, *a, **k):
  70                 super().__init__(*a, **k)
  71                 self.daemon = True
  72                 self.logger = logging.getLogger('merkleMaker')
  73                 self.CoinbasePrefix = b''
  74                 self.CoinbaseAux = {}
  75                 self.isOverflowed = False
  76                 self.lastWarning = {}
  77                 self.MinimumTxnUpdateWait = 5
  78                 self.overflowed = 0
  79                 self.DifficultyChangeMod = 2016
  80                 self.MinimumTemplateAcceptanceRatio = 0
  81                 self.MinimumTemplateScore = 1
  82                 self.currentBlock = (None, None, None)
  83                 self.lastBlock = (None, None, None)
  84
  85         def _prepare(self):
  86                 self.TemplateSources = list(getattr(self, 'TemplateSources', ()))
  87                 self.TemplateChecks = list(getattr(self, 'TemplateChecks', ()))
  88                 if getattr(self, 'BlockSubmissions', None) is None:
  89                         self.BlockSubmissions = ()
  90                 if hasattr(self, 'UpstreamURI'):
  91                         self.TemplateSources.append({
  92                                 'name': 'UpstreamURI',
  93                                 'uri': self.UpstreamURI,
  94                         })
  95                 URI2Name = {}
  96                 Name2URI = {}
  97                 for a in (self.TemplateSources + self.TemplateChecks + list(self.BlockSubmissions)):
  98                         if not ('name' in a and 'uri' in a):
  99                                 continue
 100                         URI2Name.setdefault(a['uri'], a['name'])
 101                         Name2URI.setdefault(a['name'], a['uri'])
 102                 def URINamePair(a, defname):
 103                         if 'name' not in a:
 104                                 a['name'] = URI2Name.get(a['uri'], defname)
 105                         elif 'uri' not in a:
 106                                 a['uri'] = Name2URI[a['name']]
 107                 _URI2Access = {}
 108                 def URI2Access(uri):
 109                         if uri not in _URI2Access:
 110                                 access = jsonrpc.ServiceProxy(uri)
 111                                 access.OldGMP = False
 112                                 _URI2Access[uri] = access
 113                         return _URI2Access[uri]
 114                 LeveledTS = {}
 115                 for i in range(len(self.TemplateSources)):
 116                         TS = self.TemplateSources[i]
 117                         URINamePair(TS, 'TemplateSources[%u]' % (i,))
 118                         TS.setdefault('priority', 0)
 119                         TS.setdefault('weight', 1)
 120                         TS['access'] = URI2Access(TS['uri'])
 121                         LeveledTS.setdefault(TS['priority'], []).append(TS)
 122                 LeveledTS = tuple(x[1] for x in sorted(LeveledTS.items()))
 123                 self.TemplateSources = LeveledTS
 124                 for i in range(len(self.TemplateChecks)):
 125                         TC = self.TemplateChecks[i]
 126                         URINamePair(TC, 'TemplateChecks[%u]' % (i,))
 127                         TC.setdefault('unanimous', False)
 128                         TC.setdefault('weight', 1)
 129                         TC['access'] = URI2Access(TC['uri'])
 130                 for i in range(len(getattr(self, 'BlockSubmissions', ()))):
 131                         BS = self.BlockSubmissions[i]
 132                         URINamePair(BS, 'BlockSubmissions[%u]' % (i,))
 133                         BS['access'] = URI2Access(BS['uri'])
 134
 135                 self.ready = False
 136                 self.readyCV = threading.Condition()
 137
 138                 self.currentMerkleTree = None
 139                 self.merkleRoots = deque(maxlen=self.WorkQueueSizeRegular[1])
 140                 self.LowestMerkleRoots = self.WorkQueueSizeRegular[1]
 141
 142                 if not hasattr(self, 'WorkQueueSizeClear'):
 143                         self.WorkQueueSizeClear = self.WorkQueueSizeLongpoll
 144                 self._MaxClearSize = max(self.WorkQueueSizeClear[1], self.WorkQueueSizeLongpoll[1])
 145                 self.clearMerkleRoots = Queue(self._MaxClearSize)
 146                 self.LowestClearMerkleRoots = self.WorkQueueSizeClear[1]
 147                 self.nextMerkleRoots = Queue(self._MaxClearSize)
 148
 149                 if not hasattr(self, 'WarningDelay'):
 150                         self.WarningDelay = max(15, self.MinimumTxnUpdateWait * 2)
 151                 if not hasattr(self, 'WarningDelayTxnLongpoll'):
 152                         self.WarningDelayTxnLongpoll = self.WarningDelay
 153                 if not hasattr(self, 'WarningDelayMerkleUpdate'):
 154                         self.WarningDelayMerkleUpdate = self.WarningDelay
 155
 156                 self.lastMerkleUpdate = 0
 157                 self.nextMerkleUpdate = 0
 158
 159         def createClearMerkleTree(self, height):
 160                 subsidy = 5000000000 >> (height // 210000)
 161                 cbtxn = self.makeCoinbaseTxn(subsidy, False)
 162                 cbtxn.assemble()
 163                 return MerkleTree([cbtxn])
 164
 165         def updateBlock(self, newBlock, height = None, bits = None, _HBH = None):
 166                 if newBlock == self.currentBlock[0]:
 167                         if height in (None, self.currentBlock[1]) and bits in (None, self.currentBlock[2]):
 168                                 return
 169                         if not self.currentBlock[2] is None:
 170                                 self.logger.error('Was working on block with wrong specs: %s (height: %d->%d; bits: %s->%s' % (
 171                                         b2a_hex(newBlock[::-1]).decode('utf8'),
 172                                         self.currentBlock[1],
 173                                         height,
 174                                         b2a_hex(self.currentBlock[2][::-1]).decode('utf8'),
 175                                         b2a_hex(bits[::-1]).decode('utf8'),
 176                                 ))
 177                                 if self.needMerkle == 1:
 178                                         self.needMerkle = False
 179                                 self.onBlockUpdate()
 180
 181                 # Old block is invalid
 182                 if self.currentBlock[0] != newBlock:
 183                         self.lastBlock = self.currentBlock
 184
 185                 lastHeight = self.currentBlock[1]
 186                 if height is None:
 187                         height = self.currentBlock[1] + 1
 188                 if bits is None:
 189                         if height % self.DifficultyChangeMod == 1 or self.currentBlock[2] is None:
 190                                 self.logger.warning('New block: %s (height %d; bits: UNKNOWN)' % (b2a_hex(newBlock[::-1]).decode('utf8'), height))
 191                                 # Pretend to be 1 lower height, so we possibly retain nextMerkleRoots
 192                                 self.currentBlock = (None, height - 1, None)
 193                                 OCMR = self.clearMerkleRoots
 194                                 self.clearMerkleRoots = Queue(self.WorkQueueSizeClear[1])
 195                                 if OCMR.empty():
 196                                         OCMR.put(None)
 197                                 self.merkleRoots.clear()
 198                                 self.ready = False
 199                                 return
 200                         else:
 201                                 bits = self.currentBlock[2]
 202
 203                 if _HBH is None:
 204                         _HBH = (b2a_hex(newBlock[::-1]).decode('utf8'), b2a_hex(bits[::-1]).decode('utf8'))
 205                 self.logger.info('New block: %s (height: %d; bits: %s)' % (_HBH[0], height, _HBH[1]))
 206                 self.currentBlock = (newBlock, height, bits)
 207
 208                 if lastHeight != height:
 209                         # TODO: Perhaps reuse clear merkle trees more intelligently
 210                         OCMR = self.clearMerkleRoots
 211                         if lastHeight == height - 1:
 212                                 self.curClearMerkleTree = self.nextMerkleTree
 213                                 self.clearMerkleRoots = self.nextMerkleRoots
 214                                 self.logger.debug('Adopting next-height clear merkleroots :)')
 215                         else:
 216                                 if lastHeight:
 217                                         self.logger.warning('Change from height %d->%d; no longpoll merkleroots available!' % (lastHeight, height))
 218                                 self.curClearMerkleTree = self.createClearMerkleTree(height)
 219                                 self.clearMerkleRoots = Queue(self.WorkQueueSizeClear[1])
 220                         if OCMR.empty():
 221                                 OCMR.put(None)
 222                         self.nextMerkleTree = self.createClearMerkleTree(height + 1)
 223                         self.nextMerkleRoots = Queue(self._MaxClearSize)
 224                 else:
 225                         self.logger.debug('Already using clear merkleroots for this height')
 226                 self.currentMerkleTree = self.curClearMerkleTree
 227                 self.merkleRoots.clear()
 228
 229                 if not self.ready:
 230                         self.ready = True
 231                         with self.readyCV:
 232                                 self.readyCV.notify_all()
 233
 234                 self.needMerkle = 2
 235                 self.onBlockChange()
 236
 237         def _trimBlock(self, MP, txnlist, txninfo, floodn, msgf):
 238                 fee = txninfo[-1].get('fee', None)
 239                 if fee is None:
 240                         raise self._floodCritical(now, floodn, doin=msgf('fees unknown'))
 241                 if fee:
 242                         # FIXME: coinbasevalue is *not* guaranteed to exist here
 243                         MP['coinbasevalue'] -= fee
 244
 245                 txnlist[-1:] = ()
 246                 txninfo[-1:] = ()
 247
 248                 return True
 249
 250         # Aggressive "Power Of Two": Remove transactions even with fees to reach our goal
 251         def _APOT(self, txninfopot, MP, POTInfo):
 252                 feeTxnsTrimmed = 0
 253                 feesTrimmed = 0
 254                 for txn in txninfopot:
 255                         if txn.get('fee') is None:
 256                                 self._floodWarning(now, 'APOT-No-Fees', doin='Upstream didn\'t provide fee information required for aggressive POT', logf=self.logger.info)
 257                                 return
 258                         if not txn['fee']:
 259                                 continue
 260                         feesTrimmed += txn['fee']
 261                         feeTxnsTrimmed += 1
 262                 MP['coinbasevalue'] -= feesTrimmed
 263
 264                 POTInfo[2] = [feeTxnsTrimmed, feesTrimmed]
 265                 self._floodWarning(now, 'POT-Trimming-Fees', doin='Aggressive POT trimming %d transactions with %d.%08d BTC total fees' % (feeTxnsTrimmed, feesTrimmed//100000000, feesTrimmed % 100000000), logf=self.logger.debug)
 266
 267                 return True
 268
 269         def _makeBlockSafe(self, MP, txnlist, txninfo):
 270                 blocksize = sum(map(len, txnlist)) + 80
 271                 while blocksize > 934464:  # 1 "MB" limit - 64 KB breathing room
 272                         txnsize = len(txnlist[-1])
 273                         self._trimBlock(MP, txnlist, txninfo, 'SizeLimit', lambda x: 'Making blocks over 1 MB size limit (%d bytes; %s)' % (blocksize, x))
 274                         blocksize -= txnsize
 275
 276                 # NOTE: This check doesn't work at all without BIP22 transaction obj format
 277                 blocksigops = sum(a.get('sigops', 0) for a in txninfo)
 278                 while blocksigops > 19488:  # 20k limit - 0x200 breathing room
 279                         txnsigops = txninfo[-1]['sigops']
 280                         self._trimBlock(MP, txnlist, txninfo, 'SigOpLimit', lambda x: 'Making blocks over 20k SigOp limit (%d; %s)' % (blocksigops, x))
 281                         blocksigops -= txnsigops
 282
 283                 # Aim to produce blocks with "Power Of Two" transaction counts
 284                 # This helps avoid any chance of someone abusing CVE-2012-2459 with them
 285                 POTMode = getattr(self, 'POT', 1)
 286                 txncount = len(txnlist) + 1
 287                 if POTMode:
 288                         feetxncount = txncount
 289                         for i in range(txncount - 2, -1, -1):
 290                                 if 'fee' not in txninfo[i] or txninfo[i]['fee']:
 291                                         break
 292                                 feetxncount -= 1
 293
 294                         if getattr(self, 'Greedy', None):
 295                                 # Aim to cut off extra zero-fee transactions on the end
 296                                 # NOTE: not cutting out ones intermixed, in case of dependencies
 297                                 idealtxncount = feetxncount
 298                         else:
 299                                 idealtxncount = txncount
 300
 301                         pot = 2**int(log(idealtxncount, 2))
 302                         POTInfo = MP['POTInfo'] = [[idealtxncount, feetxncount, txncount], [pot, None], None]
 303                         if pot < idealtxncount:
 304                                 if pot * 2 <= txncount:
 305                                         pot *= 2
 306                                 elif pot >= feetxncount:
 307                                         pass
 308                                 elif POTMode > 1:
 309                                         if self._APOT(txninfo[pot-1:], MP, POTInfo):
 310                                                 # Trimmed even transactions with fees
 311                                                 pass
 312                                         else:
 313                                                 pot = idealtxncount
 314                                                 self._floodWarning(now, 'Non-POT', doin='Making merkle tree with %d transactions (ideal: %d; max: %d)' % (pot, idealtxncount, txncount))
 315                                 else:
 316                                         pot = idealtxncount
 317                         POTInfo[1][1] = pot
 318                         pot -= 1
 319                         txnlist[pot:] = ()
 320                         txninfo[pot:] = ()
 321
 322         def _CallGBT(self, TS):
 323                 access = TS['access']
 324                 self.logger.debug('Requesting new template from \'%s\'' % (TS['name'],))
 325                 try:
 326                         # First, try BIP 22 standard getblocktemplate :)
 327                         MP = access.getblocktemplate(self.GBTReq)
 328                         access.OldGMP = False
 329                 except:
 330                         try:
 331                                 # Failing that, give BIP 22 draft (2012-02 through 2012-07) getmemorypool a chance
 332                                 MP = access.getmemorypool(self.GMPReq)
 333                         except:
 334                                 try:
 335                                         # Finally, fall back to bitcoind 0.5/0.6 getmemorypool
 336                                         MP = access.getmemorypool()
 337                                 except:
 338                                         MP = False
 339                         if MP is False:
 340                                 # This way, we get the error from the BIP22 call if the old one fails too
 341                                 raise
 342
 343                         # Pre-BIP22 server (bitcoind <0.7 or Eloipool <20120513)
 344                         if not access.OldGMP:
 345                                 access.OldGMP = True
 346                                 self.logger.warning('Upstream \'%s\' is not BIP 22 compatible' % (TS['name'],))
 347
 348                 return MP
 349
 350         def _ProcessGBT(self, MP, TS = None):
 351                 oMP = MP
 352                 MP = deepcopy(MP)
 353
 354                 prevBlock = bytes.fromhex(MP['previousblockhash'])[::-1]
 355                 if 'height' not in MP:
 356                         MP['height'] = TS['access'].getinfo()['blocks'] + 1
 357                 height = MP['height']
 358                 bits = bytes.fromhex(MP['bits'])[::-1]
 359                 (MP['_bits'], MP['_prevBlock']) = (bits, prevBlock)
 360                 if (prevBlock, height, bits) != self.currentBlock and (self.currentBlock[1] is None or height > self.currentBlock[1]):
 361                         self.updateBlock(prevBlock, height, bits, _HBH=(MP['previousblockhash'], MP['bits']))
 362
 363                 txnlist = MP['transactions']
 364                 if len(txnlist) and isinstance(txnlist[0], dict):
 365                         txninfo = txnlist
 366                         txnlist = tuple(a['data'] for a in txnlist)
 367                 elif 'transactionfees' in MP:
 368                         # Backward compatibility with pre-BIP22 gmp_fees branch
 369                         txninfo = [{'fee':a} for a in MP['transactionfees']]
 370                 else:
 371                         # Backward compatibility with pre-BIP22 hex-only (bitcoind <0.7, Eloipool <future)
 372                         txninfo = [{}] * len(txnlist)
 373                 # TODO: cache Txn or at least txid from previous merkle roots?
 374                 txnlist = [a for a in map(bytes.fromhex, txnlist)]
 375
 376                 self._makeBlockSafe(MP, txnlist, txninfo)
 377
 378                 cbtxn = self.makeCoinbaseTxn(MP['coinbasevalue'])
 379                 cbtxn.setCoinbase(b'\0\0')
 380                 cbtxn.assemble()
 381                 txnlist.insert(0, cbtxn.data)
 382                 txninfo.insert(0, {
 383                 })
 384
 385                 txnlist = [a for a in map(Txn, txnlist[1:])]
 386                 txnlist.insert(0, cbtxn)
 387                 txnlist = list(txnlist)
 388                 newMerkleTree = MerkleTree(txnlist)
 389                 newMerkleTree.POTInfo = MP.get('POTInfo')
 390                 newMerkleTree.MP = MP
 391                 newMerkleTree.oMP = oMP
 392
 393                 return newMerkleTree
 394
 395         def _CheckTemplate(self, newMerkleTree, TS):
 396                 TCList = self.TemplateChecks
 397                 if not TCList:
 398                         if 'proposal' not in newMerkleTree.oMP.get('capabilities', ()):
 399                                 return (None, None)
 400                         TCList = (
 401                                 {
 402                                         'name': TS['name'],
 403                                         'access': TS['access'],
 404                                         'unanimous': True,
 405                                         'weight': 1,
 406                                 },
 407                         )
 408
 409                 MP = newMerkleTree.MP
 410                 (prevBlock, height, bits) = (MP['_prevBlock'], MP['height'], MP['_bits'])
 411                 txnlist = newMerkleTree.data
 412                 cbtxn = txnlist[0]
 413
 414                 coinbase = self.makeCoinbase(height=height)
 415                 cbtxn.setCoinbase(coinbase)
 416                 cbtxn.assemble()
 417                 merkleRoot = newMerkleTree.merkleRoot()
 418                 MRD = (merkleRoot, newMerkleTree, coinbase, prevBlock, bits)
 419                 blkhdr = MakeBlockHeader(MRD)
 420                 data = assembleBlock(blkhdr, txnlist)
 421                 ProposeReq = {
 422                         "mode": "proposal",
 423                         "data": b2a_hex(data).decode('utf8'),
 424                 }
 425
 426                 AcceptedScore = 0
 427                 RejectedScore = 0
 428                 Rejections = {}
 429                 ProposalErrors = {}
 430                 for TC in TCList:
 431                         caccess = TC['access']
 432                         try:
 433                                 propose = caccess.getblocktemplate(ProposeReq)
 434                         except (socket.error, ValueError) as e:
 435                                 self.logger.error('Upstream \'%s\' errored on proposal from \'%s\': %s' % (TC['name'], TS['name'], e))
 436                                 ProposalErrors[TC['name']] = e
 437                                 continue
 438                         if propose is None:
 439                                 AcceptedScore += TC['weight']
 440                                 self.logger.debug('Upstream \'%s\' accepted proposal' % (TC['name'],))
 441                         elif propose == 'orphan':
 442                                 self.logger.debug('Upstream \'%s\' considered proposal an orphan' % (TC['name'],))
 443                                 ProposalErrors[TC['name']] = propose
 444                         else:
 445                                 RejectedScore += TC['weight']
 446                                 Rejections[TC['name']] = propose
 447                                 try:
 448                                         propose = propose['reject-reason']
 449                                 except:
 450                                         pass
 451                                 self.logger.error('Upstream \'%s\' rejected proposed block from \'%s\': %s' % (TC['name'], TS['name'], propose))
 452
 453                 if Rejections:
 454                         RPInfo = {
 455                                 'merkleTree': newMerkleTree,
 456                                 'AcceptedScore': AcceptedScore,
 457                                 'RejectedScore': RejectedScore,
 458                                 'Rejections': Rejections,
 459                                 'ProposalErrors': ProposalErrors,
 460                         }
 461                         self.RejectedProposal = RPInfo
 462
 463                         try:
 464                                 global _filecounter
 465                                 _filecounter += 1
 466                                 import pickle
 467                                 with open('RejectedProposals/%d_%d' % (int(time()), _filecounter), 'wb') as f:
 468                                         pickle.dump(RPInfo, f)
 469                         except IOError:
 470                                 pass
 471
 472                 TotalScore = AcceptedScore + RejectedScore
 473
 474                 return (AcceptedScore, TotalScore)
 475
 476         def _updateMerkleTree_fromTS(self, TS):
 477                 MP = self._CallGBT(TS)
 478                 newMerkleTree = self._ProcessGBT(MP, TS)
 479
 480                 # Some versions of bitcoinrpc ServiceProxy have problems copying/pickling, so just store name and URI for now
 481                 newMerkleTree.source = TS['name']
 482                 newMerkleTree.source_uri = TS['uri']
 483
 484                 (AcceptedScore, TotalScore) = self._CheckTemplate(newMerkleTree, TS)
 485                 if TotalScore is None:
 486                         return (0, newMerkleTree)
 487
 488                 if TotalScore:
 489                         AcceptRatio = AcceptedScore / TotalScore
 490                 else:
 491                         AcceptRatio = 0.0
 492
 493                 self.logger.debug('Template from \'%s\' has %s acceptance ratio and score of %s' % (TS['name'], AcceptRatio, AcceptedScore))
 494
 495                 if AcceptRatio <= self.MinimumTemplateAcceptanceRatio:
 496                         return None
 497
 498                 if TotalScore < self.MinimumTemplateScore:
 499                         return None
 500
 501                 return (AcceptRatio, newMerkleTree)
 502
 503         def _updateMerkleTree_I(self):
 504                 Best = (-1, None)
 505                 for TSPriList in self.TemplateSources:
 506                         # FIXME: Implement weighting
 507                         for i in range(len(TSPriList)):
 508                                 TS = TSPriList.pop(0)
 509                                 TSPriList.append(TS)
 510
 511                                 try:
 512                                         r = self._updateMerkleTree_fromTS(TS)
 513                                         if r is None:
 514                                                 # Failed completely
 515                                                 continue
 516
 517                                         (AcceptRatio, newMerkleTree) = r
 518
 519                                         # NOTE: If you're going to try to remove this preference for the highest block, you need to (at least) stop _ProcessGBT from calling updateBlock whenever it sees a new high
 520                                         AcceptRatio += newMerkleTree.MP['height']
 521
 522                                         if Best[0] < AcceptRatio:
 523                                                 Best = r
 524                                                 if AcceptRatio == 1:
 525                                                         break
 526                                 except:
 527                                         if TSPriList == self.TemplateSources[-1] and i == len(TSPriList) - 1 and Best[1] is None:
 528                                                 raise
 529                                         else:
 530                                                 self.logger.error(traceback.format_exc())
 531
 532                 BestMT = Best[1]
 533                 if BestMT is None:
 534                         raise RuntimeError('Failed to create usable template')
 535
 536                 self.logger.debug('Updating merkle tree with template from \'%s\'' % (BestMT.source,))
 537                 MP = BestMT.MP
 538                 blkbasics = (MP['_prevBlock'], MP['height'], MP['_bits'])
 539                 if blkbasics != self.currentBlock:
 540                         self.updateBlock(*blkbasics, _HBH=(MP['previousblockhash'], MP['bits']))
 541                 self.currentMerkleTree = BestMT
 542
 543         def _updateMerkleTree(self):
 544                 global now
 545                 self.logger.debug('Polling for new block template')
 546                 self.nextMerkleUpdate = now + self.TxnUpdateRetryWait
 547
 548                 self._updateMerkleTree_I()
 549
 550                 self.lastMerkleUpdate = now
 551                 self.nextMerkleUpdate = now + self.MinimumTxnUpdateWait
 552
 553                 if self.needMerkle == 2:
 554                         self.needMerkle = 1
 555                         self.needMerkleSince = now
 556
 557         def updateMerkleTree(self):
 558                 global now
 559                 now = time()
 560                 self._updateMerkleTree()
 561
 562         def makeCoinbase(self, height):
 563                 now = int(time())
 564                 if now > _makeCoinbase[0]:
 565                         _makeCoinbase[0] = now
 566                         _makeCoinbase[1] = 0
 567                 else:
 568                         _makeCoinbase[1] += 1
 569                 rv = self.CoinbasePrefix
 570                 rv += pack('>L', now) + pack('>Q', _makeCoinbase[1]).lstrip(b'\0')
 571                 # NOTE: Not using varlenEncode, since this is always guaranteed to be < 100
 572                 rv = bytes( (len(rv),) ) + rv
 573                 for v in self.CoinbaseAux.values():
 574                         rv += v
 575                 if len(rv) > 95:
 576                         t = time()
 577                         if self.overflowed < t - 300:
 578                                 self.logger.warning('Overflowing coinbase data! %d bytes long' % (len(rv),))
 579                                 self.overflowed = t
 580                                 self.isOverflowed = True
 581                         rv = rv[:95]
 582                 else:
 583                         self.isOverflowed = False
 584                 rv = bitcoin.script.encodeUNum(height) + rv
 585                 return rv
 586
 587         def makeMerkleRoot(self, merkleTree, height):
 588                 cbtxn = merkleTree.data[0]
 589                 cb = self.makeCoinbase(height=height)
 590                 cbtxn.setCoinbase(cb)
 591                 cbtxn.assemble()
 592                 merkleRoot = merkleTree.merkleRoot()
 593                 return (merkleRoot, merkleTree, cb)
 594
 595         _doing_last = None
 596         def _doing(self, what):
 597                 if self._doing_last == what:
 598                         self._doing_i += 1
 599                         return
 600                 global now
 601                 if self._doing_last:
 602                         self.logger.debug("Switching from (%4dx in %5.3f seconds) %s => %s" % (self._doing_i, now - self._doing_s, self._doing_last, what))
 603                 self._doing_last = what
 604                 self._doing_i = 1
 605                 self._doing_s = now
 606
 607         def _floodWarning(self, now, wid, wmsgf = None, doin = True, logf = None):
 608                 if doin is True:
 609                         doin = self._doing_last
 610                         def a(f = wmsgf):
 611                                 return lambda: "%s (doing %s)" % (f(), doin)
 612                         wmsgf = a()
 613                 winfo = self.lastWarning.setdefault(wid, [0, None])
 614                 (lastTime, lastDoing) = winfo
 615                 if now <= lastTime + max(5, self.MinimumTxnUpdateWait):
 616                         return
 617                 winfo[0] = now
 618                 nowDoing = doin
 619                 winfo[1] = nowDoing
 620                 if logf is None:
 621                         logf = self.logger.warning
 622                 logf(wmsgf() if wmsgf else doin)
 623
 624         def _makeOne(self, putf, merkleTree, height):
 625                 MakingAtThisHeight = self.currentBlock[1]
 626                 MR = self.makeMerkleRoot(merkleTree, height=height)
 627                 # Only add it if the height hasn't changed in the meantime, to avoid a race
 628                 if self.currentBlock[1] == MakingAtThisHeight:
 629                         putf(MR)
 630
 631         def makeClear(self):
 632                 self._doing('clear merkle roots')
 633                 self._makeOne(self.clearMerkleRoots.put, self.curClearMerkleTree, height=self.currentBlock[1])
 634
 635         def makeNext(self):
 636                 self._doing('longpoll merkle roots')
 637                 self._makeOne(self.nextMerkleRoots.put, self.nextMerkleTree, height=self.currentBlock[1] + 1)
 638
 639         def makeRegular(self):
 640                 self._doing('regular merkle roots')
 641                 self._makeOne(self.merkleRoots.append, self.currentMerkleTree, height=self.currentBlock[1])
 642
 643         def merkleMaker_II(self):
 644                 global now
 645
 646                 # No bits = no mining :(
 647                 if not self.ready:
 648                         return self._updateMerkleTree()
 649
 650                 # First, ensure we have the minimum clear, next, and regular (in that order)
 651                 if self.clearMerkleRoots.qsize() < self.WorkQueueSizeClear[0]:
 652                         return self.makeClear()
 653                 if self.nextMerkleRoots.qsize() < self.WorkQueueSizeLongpoll[0]:
 654                         return self.makeNext()
 655                 if len(self.merkleRoots) < self.WorkQueueSizeRegular[0]:
 656                         return self.makeRegular()
 657
 658                 # If we've met the minimum requirements, consider updating the merkle tree
 659                 if self.nextMerkleUpdate <= now:
 660                         return self._updateMerkleTree()
 661
 662                 # Finally, fill up clear, next, and regular until we've met the maximums
 663                 if self.clearMerkleRoots.qsize() < self.WorkQueueSizeClear[1]:
 664                         return self.makeClear()
 665                 if self.nextMerkleRoots.qsize() < self.WorkQueueSizeLongpoll[1]:
 666                         return self.makeNext()
 667                 if len(self.merkleRoots) < self.WorkQueueSizeRegular[1] or self.merkleRoots[0][1] != self.currentMerkleTree:
 668                         return self.makeRegular()
 669
 670                 # Nothing left to do, fire onBlockUpdate event (if appropriate) and sleep
 671                 if self.needMerkle == 1:
 672                         self.onBlockUpdate()
 673                         self.needMerkle = False
 674                 self._doing('idle')
 675                 # TODO: rather than sleepspin, block until MinimumTxnUpdateWait expires or threading.Condition(?)
 676                 sleep(self.IdleSleepTime)
 677
 678         def merkleMaker_I(self):
 679                 global now
 680                 now = time()
 681
 682                 self.merkleMaker_II()
 683
 684                 if self.needMerkle == 1 and now > self.needMerkleSince + self.WarningDelayTxnLongpoll:
 685                         self._floodWarning(now, 'NeedMerkle', lambda: 'Transaction-longpoll requested %d seconds ago, and still not ready. Is your server fast enough to keep up with your configured WorkQueueSizeRegular maximum?' % (now - self.needMerkleSince,))
 686                 if now > self.nextMerkleUpdate + self.WarningDelayMerkleUpdate:
 687                         self._floodWarning(now, 'MerkleUpdate', lambda: "Haven't updated the merkle tree in at least %d seconds! Is your server fast enough to keep up with your configured work queue minimums?" % (now - self.lastMerkleUpdate,))
 688
 689         def run(self):
 690                 while True:
 691                         try:
 692                                 self.merkleMaker_I()
 693                         except:
 694                                 self.logger.critical(traceback.format_exc())
 695
 696         def start(self, *a, **k):
 697                 self._prepare()
 698                 super().start(*a, **k)
 699
 700         def getMRD(self):
 701                 try:
 702                         MRD = self.merkleRoots.pop()
 703                         self.LowestMerkleRoots = min(len(self.merkleRoots), self.LowestMerkleRoots)
 704                         rollPrevBlk = False
 705                 except IndexError:
 706                         qsz = self.clearMerkleRoots.qsize()
 707                         if qsz < 0x10:
 708                                 self.logger.warning('clearMerkleRoots running out! only %d left' % (qsz,))
 709                         MRD = None
 710                         while MRD is None:
 711                                 MRD = self.clearMerkleRoots.get()
 712                         self.LowestClearMerkleRoots = min(self.clearMerkleRoots.qsize(), self.LowestClearMerkleRoots)
 713                         rollPrevBlk = True
 714                 (merkleRoot, merkleTree, cb) = MRD
 715                 (prevBlock, height, bits) = self.currentBlock
 716                 return (merkleRoot, merkleTree, cb, prevBlock, bits, rollPrevBlk)
 717
 718         def getMC(self, wantClear = False):
 719                 if not self.ready:
 720                         with self.readyCV:
 721                                 while not self.ready:
 722                                         self.readyCV.wait()
 723                 (prevBlock, height, bits) = self.currentBlock
 724                 mt = self.curClearMerkleTree if wantClear else self.currentMerkleTree
 725                 cb = self.makeCoinbase(height=height)
 726                 rollPrevBlk = (mt == self.curClearMerkleTree)
 727                 return (height, mt, cb, prevBlock, bits, rollPrevBlk)
 728
 729 # merkleMaker tests
 730 def _test():
 731         global now
 732         now = 1337039788
 733         MM = merkleMaker()
 734         reallogger = MM.logger
 735         class fakelogger:
 736                 LO = False
 737                 def critical(self, *a):
 738                         if self.LO > 1: return
 739                         reallogger.critical(*a)
 740                 def warning(self, *a):
 741                         if self.LO: return
 742                         reallogger.warning(*a)
 743                 def debug(self, *a):
 744                         pass
 745         MM.logger = fakelogger()
 746         class NMTClass:
 747                 pass
 748
 749         # _makeBlockSafe tests
 750         from copy import deepcopy
 751         MP = {
 752                 'coinbasevalue':50,
 753         }
 754         txnlist = [b'\0', b'\x01', b'\x02']
 755         txninfo = [{'fee':0, 'sigops':1}, {'fee':5, 'sigops':10000}, {'fee':0, 'sigops':10001}]
 756         def MBS(LO = 0):
 757                 m = deepcopy( (MP, txnlist, txninfo) )
 758                 MM.logger.LO = LO
 759                 try:
 760                         MM._makeBlockSafe(*m)
 761                 except:
 762                         if LO < 2:
 763                                 raise
 764                 else:
 765                         assert LO < 2  # An expected error wasn't thrown
 766                 if 'POTInfo' in m[0]:
 767                         del m[0]['POTInfo']
 768                 return m
 769         MM.POT = 0
 770         assert MBS() == (MP, txnlist[:2], txninfo[:2])
 771         txninfo[2]['fee'] = 1
 772         MPx = deepcopy(MP)
 773         MPx['coinbasevalue'] -= 1
 774         assert MBS() == (MPx, txnlist[:2], txninfo[:2])
 775         txninfo[2]['sigops'] = 1
 776         assert MBS(1) == (MP, txnlist, txninfo)
 777         # APOT tests
 778         MM.POT = 2
 779         txnlist.append(b'\x03')
 780         txninfo.append({'fee':1, 'sigops':0})
 781         MPx = deepcopy(MP)
 782         MPx['coinbasevalue'] -= 1
 783         assert MBS() == (MPx, txnlist[:3], txninfo[:3])
 784         # POT tests
 785         MM.POT = 1
 786         MM.Greedy = True
 787         txninfo[1]['fee'] = 0
 788         txninfo[2]['fee'] = 0
 789         assert MBS(1) == (MP, txnlist, txninfo)
 790         # _ProcessGBT tests
 791         def makeCoinbaseTxn(coinbaseValue, useCoinbaser = True):
 792                 txn = Txn.new()
 793                 txn.addOutput(coinbaseValue, b'')
 794                 return txn
 795         MM.makeCoinbaseTxn = makeCoinbaseTxn
 796         MM.updateBlock = lambda *a, **ka: None
 797         gbt = {
 798                 'transactions': [
 799                         {'data': '11', 'depends': [], 'fee': 1, 'sigops': 1},
 800                         {'data': '11', 'depends': [], 'fee': 0, 'sigops': 1},
 801                         {'data': '11', 'depends': [], 'fee': 0, 'sigops': 1},
 802                         {'data': '11', 'depends': [], 'fee': 1, 'sigops': 2}
 803                 ],
 804                 'height': 219507,
 805                 'coinbasevalue': 3,
 806                 'previousblockhash': '000000000000012806bc100006dc83220bd9c2ac2709dc14a0d0fa1d6f9b733c',
 807                 'version': 1,
 808                 'bits': '1a05a6b1'
 809         }
 810         nMT = MM._ProcessGBT(gbt)
 811         assert len(nMT.data) == 5
 812         nMT.data[0].disassemble()
 813         assert sum(outp[0] for outp in nMT.data[0].outputs) == 3
 814         MM.POT = 2
 815         nMT = MM._ProcessGBT(gbt)
 816         assert len(nMT.data) in (2, 4)
 817         nMT.data[0].disassemble()
 818         assert sum(outp[0] for outp in nMT.data[0].outputs) == 2
 819
 820 _test()