merklemaker.py

   1 # Eloipool - Python Bitcoin pool server
   2 # Copyright (C) 2011-2012  Luke Dashjr <luke-jr+eloipool@utopios.org>
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU Affero General Public License as
   6 # published by the Free Software Foundation, either version 3 of the
   7 # License, or (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU Affero General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU Affero General Public License
  15 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 from binascii import b2a_hex
  18 import bitcoin.script
  19 from bitcoin.script import countSigOps
  20 from bitcoin.txn import Txn
  21 from bitcoin.varlen import varlenEncode, varlenDecode
  22 from collections import deque
  23 from copy import deepcopy
  24 from queue import Queue
  25 import jsonrpc
  26 import logging
  27 from math import log
  28 from merkletree import MerkleTree
  29 import socket
  30 from struct import pack
  31 import threading
  32 from time import sleep, time
  33 import traceback
  34
  35 _makeCoinbase = [0, 0]
  36 _filecounter = 0
  37
  38 def MakeBlockHeader(MRD):
  39         (merkleRoot, merkleTree, coinbase, prevBlock, bits) = MRD[:5]
  40         timestamp = pack('<L', int(time()))
  41         hdr = b'\2\0\0\0' + prevBlock + merkleRoot + timestamp + bits + b'iolE'
  42         return hdr
  43
  44 def assembleBlock(blkhdr, txlist):
  45         payload = blkhdr
  46         payload += varlenEncode(len(txlist))
  47         for tx in txlist:
  48                 payload += tx.data
  49         return payload
  50
  51 class merkleMaker(threading.Thread):
  52         GBTCaps = [
  53                 'coinbasevalue',
  54                 'coinbase/append',
  55                 'coinbase',
  56                 'generation',
  57                 'time',
  58                 'transactions/remove',
  59                 'prevblock',
  60         ]
  61         GBTReq = {
  62                 'capabilities': GBTCaps,
  63         }
  64         GMPReq = {
  65                 'capabilities': GBTCaps,
  66                 'tx': 'obj',
  67         }
  68
  69         def __init__(self, *a, **k):
  70                 super().__init__(*a, **k)
  71                 self.daemon = True
  72                 self.logger = logging.getLogger('merkleMaker')
  73                 self.CoinbasePrefix = b''
  74                 self.CoinbaseAux = {}
  75                 self.isOverflowed = False
  76                 self.lastWarning = {}
  77                 self.MinimumTxnUpdateWait = 5
  78                 self.overflowed = 0
  79                 self.DifficultyChangeMod = 2016
  80                 self.MinimumTemplateAcceptanceRatio = 0
  81                 self.MinimumTemplateScore = 1
  82                 self.currentBlock = (None, None, None)
  83                 self.lastBlock = (None, None, None)
  84
  85         def _prepare(self):
  86                 self.TemplateSources = list(getattr(self, 'TemplateSources', ()))
  87                 self.TemplateChecks = list(getattr(self, 'TemplateChecks', ()))
  88                 if getattr(self, 'BlockSubmissions', None) is None:
  89                         self.BlockSubmissions = ()
  90                 if hasattr(self, 'UpstreamURI'):
  91                         self.TemplateSources.append({
  92                                 'name': 'UpstreamURI',
  93                                 'uri': self.UpstreamURI,
  94                         })
  95                 URI2Name = {}
  96                 Name2URI = {}
  97                 for a in (self.TemplateSources + self.TemplateChecks + list(self.BlockSubmissions)):
  98                         if not ('name' in a and 'uri' in a):
  99                                 continue
 100                         URI2Name.setdefault(a['uri'], a['name'])
 101                         Name2URI.setdefault(a['name'], a['uri'])
 102                 def URINamePair(a, defname):
 103                         if 'name' not in a:
 104                                 a['name'] = URI2Name.get(a['uri'], defname)
 105                         elif 'uri' not in a:
 106                                 a['uri'] = Name2URI[a['name']]
 107                 _URI2Access = {}
 108                 def URI2Access(uri):
 109                         if uri not in _URI2Access:
 110                                 access = jsonrpc.ServiceProxy(uri)
 111                                 access.OldGMP = False
 112                                 _URI2Access[uri] = access
 113                         return _URI2Access[uri]
 114                 LeveledTS = {}
 115                 for i in range(len(self.TemplateSources)):
 116                         TS = self.TemplateSources[i]
 117                         URINamePair(TS, 'TemplateSources[%u]' % (i,))
 118                         TS.setdefault('priority', 0)
 119                         TS.setdefault('weight', 1)
 120                         TS['access'] = URI2Access(TS['uri'])
 121                         LeveledTS.setdefault(TS['priority'], []).append(TS)
 122                 LeveledTS = tuple(x[1] for x in sorted(LeveledTS.items()))
 123                 self.TemplateSources = LeveledTS
 124                 for i in range(len(self.TemplateChecks)):
 125                         TC = self.TemplateChecks[i]
 126                         URINamePair(TC, 'TemplateChecks[%u]' % (i,))
 127                         TC.setdefault('unanimous', False)
 128                         TC.setdefault('weight', 1)
 129                         TC['access'] = URI2Access(TC['uri'])
 130                 for i in range(len(getattr(self, 'BlockSubmissions', ()))):
 131                         BS = self.BlockSubmissions[i]
 132                         URINamePair(BS, 'BlockSubmissions[%u]' % (i,))
 133                         BS['access'] = URI2Access(BS['uri'])
 134
 135                 self.ready = False
 136                 self.readyCV = threading.Condition()
 137
 138                 self.currentMerkleTree = None
 139                 self.merkleRoots = deque(maxlen=self.WorkQueueSizeRegular[1])
 140                 self.LowestMerkleRoots = self.WorkQueueSizeRegular[1]
 141
 142                 if not hasattr(self, 'WorkQueueSizeClear'):
 143                         self.WorkQueueSizeClear = self.WorkQueueSizeLongpoll
 144                 self._MaxClearSize = max(self.WorkQueueSizeClear[1], self.WorkQueueSizeLongpoll[1])
 145                 self.clearMerkleRoots = Queue(self._MaxClearSize)
 146                 self.LowestClearMerkleRoots = self.WorkQueueSizeClear[1]
 147                 self.nextMerkleRoots = Queue(self._MaxClearSize)
 148
 149                 if not hasattr(self, 'WarningDelay'):
 150                         self.WarningDelay = max(15, self.MinimumTxnUpdateWait * 2)
 151                 if not hasattr(self, 'WarningDelayTxnLongpoll'):
 152                         self.WarningDelayTxnLongpoll = self.WarningDelay
 153                 if not hasattr(self, 'WarningDelayMerkleUpdate'):
 154                         self.WarningDelayMerkleUpdate = self.WarningDelay
 155
 156                 self.lastMerkleUpdate = 0
 157                 self.nextMerkleUpdate = 0
 158
 159         def createClearMerkleTree(self, height):
 160                 subsidy = 5000000000 >> (height // 210000)
 161                 cbtxn = self.makeCoinbaseTxn(subsidy, False)
 162                 cbtxn.assemble()
 163                 return MerkleTree([cbtxn])
 164
 165         def updateBlock(self, newBlock, height = None, bits = None, _HBH = None):
 166                 if newBlock == self.currentBlock[0]:
 167                         if height in (None, self.currentBlock[1]) and bits in (None, self.currentBlock[2]):
 168                                 return
 169                         if not self.currentBlock[2] is None:
 170                                 self.logger.error('Was working on block with wrong specs: %s (height: %d->%d; bits: %s->%s' % (
 171                                         b2a_hex(newBlock[::-1]).decode('utf8'),
 172                                         self.currentBlock[1],
 173                                         height,
 174                                         b2a_hex(self.currentBlock[2][::-1]).decode('utf8'),
 175                                         b2a_hex(bits[::-1]).decode('utf8'),
 176                                 ))
 177                                 if self.needMerkle == 1:
 178                                         self.needMerkle = False
 179                                 self.onBlockUpdate()
 180
 181                 # Old block is invalid
 182                 if self.currentBlock[0] != newBlock:
 183                         self.lastBlock = self.currentBlock
 184
 185                 lastHeight = self.currentBlock[1]
 186                 if height is None:
 187                         height = self.currentBlock[1] + 1
 188                 if bits is None:
 189                         if height % self.DifficultyChangeMod == 1 or self.currentBlock[2] is None:
 190                                 self.logger.warning('New block: %s (height %d; bits: UNKNOWN)' % (b2a_hex(newBlock[::-1]).decode('utf8'), height))
 191                                 # Pretend to be 1 lower height, so we possibly retain nextMerkleRoots
 192                                 self.currentBlock = (None, height - 1, None)
 193                                 OCMR = self.clearMerkleRoots
 194                                 self.clearMerkleRoots = Queue(self.WorkQueueSizeClear[1])
 195                                 if OCMR.empty():
 196                                         OCMR.put(None)
 197                                 self.merkleRoots.clear()
 198                                 self.ready = False
 199                                 return
 200                         else:
 201                                 bits = self.currentBlock[2]
 202
 203                 if _HBH is None:
 204                         _HBH = (b2a_hex(newBlock[::-1]).decode('utf8'), b2a_hex(bits[::-1]).decode('utf8'))
 205                 self.logger.info('New block: %s (height: %d; bits: %s)' % (_HBH[0], height, _HBH[1]))
 206                 self.currentBlock = (newBlock, height, bits)
 207
 208                 if lastHeight != height:
 209                         # TODO: Perhaps reuse clear merkle trees more intelligently
 210                         OCMR = self.clearMerkleRoots
 211                         if lastHeight == height - 1:
 212                                 self.curClearMerkleTree = self.nextMerkleTree
 213                                 self.clearMerkleRoots = self.nextMerkleRoots
 214                                 self.logger.debug('Adopting next-height clear merkleroots :)')
 215                         else:
 216                                 if lastHeight:
 217                                         self.logger.warning('Change from height %d->%d; no longpoll merkleroots available!' % (lastHeight, height))
 218                                 self.curClearMerkleTree = self.createClearMerkleTree(height)
 219                                 self.clearMerkleRoots = Queue(self.WorkQueueSizeClear[1])
 220                         if OCMR.empty():
 221                                 OCMR.put(None)
 222                         self.nextMerkleTree = self.createClearMerkleTree(height + 1)
 223                         self.nextMerkleRoots = Queue(self._MaxClearSize)
 224                 else:
 225                         self.logger.debug('Already using clear merkleroots for this height')
 226                 self.currentMerkleTree = self.curClearMerkleTree
 227                 self.merkleRoots.clear()
 228
 229                 if not self.ready:
 230                         self.ready = True
 231                         with self.readyCV:
 232                                 self.readyCV.notify_all()
 233
 234                 self.needMerkle = 2
 235                 self.onBlockChange()
 236
 237         def _trimBlock(self, MP, txnlist, txninfo, floodn, msgf):
 238                 fee = txninfo[-1].get('fee', None)
 239                 if fee is None:
 240                         raise self._floodCritical(now, floodn, doin=msgf('fees unknown'))
 241                 if fee:
 242                         # FIXME: coinbasevalue is *not* guaranteed to exist here
 243                         MP['coinbasevalue'] -= fee
 244
 245                 txnlist[-1:] = ()
 246                 txninfo[-1:] = ()
 247
 248                 return True
 249
 250         # Aggressive "Power Of Two": Remove transactions even with fees to reach our goal
 251         def _APOT(self, txninfopot, MP, POTInfo):
 252                 feeTxnsTrimmed = 0
 253                 feesTrimmed = 0
 254                 for txn in txninfopot:
 255                         if txn.get('fee') is None:
 256                                 self._floodWarning(now, 'APOT-No-Fees', doin='Upstream didn\'t provide fee information required for aggressive POT', logf=self.logger.info)
 257                                 return
 258                         if not txn['fee']:
 259                                 continue
 260                         feesTrimmed += txn['fee']
 261                         feeTxnsTrimmed += 1
 262                 MP['coinbasevalue'] -= feesTrimmed
 263
 264                 POTInfo[2] = [feeTxnsTrimmed, feesTrimmed]
 265                 self._floodWarning(now, 'POT-Trimming-Fees', doin='Aggressive POT trimming %d transactions with %d.%08d BTC total fees' % (feeTxnsTrimmed, feesTrimmed//100000000, feesTrimmed % 100000000), logf=self.logger.debug)
 266
 267                 return True
 268
 269         def _makeBlockSafe(self, MP, txnlist, txninfo):
 270                 blocksize = sum(map(len, txnlist)) + 80
 271                 while blocksize > 934464:  # 1 "MB" limit - 64 KB breathing room
 272                         txnsize = len(txnlist[-1])
 273                         self._trimBlock(MP, txnlist, txninfo, 'SizeLimit', lambda x: 'Making blocks over 1 MB size limit (%d bytes; %s)' % (blocksize, x))
 274                         blocksize -= txnsize
 275
 276                 # NOTE: This check doesn't work at all without BIP22 transaction obj format
 277                 blocksigops = sum(a.get('sigops', 0) for a in txninfo)
 278                 while blocksigops > 19488:  # 20k limit - 0x200 breathing room
 279                         txnsigops = txninfo[-1]['sigops']
 280                         self._trimBlock(MP, txnlist, txninfo, 'SigOpLimit', lambda x: 'Making blocks over 20k SigOp limit (%d; %s)' % (blocksigops, x))
 281                         blocksigops -= txnsigops
 282
 283                 # Aim to produce blocks with "Power Of Two" transaction counts
 284                 # This helps avoid any chance of someone abusing CVE-2012-2459 with them
 285                 POTMode = getattr(self, 'POT', 1)
 286                 txncount = len(txnlist) + 1
 287                 if POTMode:
 288                         feetxncount = txncount
 289                         for i in range(txncount - 2, -1, -1):
 290                                 if 'fee' not in txninfo[i] or txninfo[i]['fee']:
 291                                         break
 292                                 feetxncount -= 1
 293
 294                         if getattr(self, 'Greedy', None):
 295                                 # Aim to cut off extra zero-fee transactions on the end
 296                                 # NOTE: not cutting out ones intermixed, in case of dependencies
 297                                 idealtxncount = feetxncount
 298                         else:
 299                                 idealtxncount = txncount
 300
 301                         pot = 2**int(log(idealtxncount, 2))
 302                         POTInfo = MP['POTInfo'] = [[idealtxncount, feetxncount, txncount], [pot, None], None]
 303                         if pot < idealtxncount:
 304                                 if pot * 2 <= txncount:
 305                                         pot *= 2
 306                                 elif pot >= feetxncount:
 307                                         pass
 308                                 elif POTMode > 1 and self._APOT(txninfo[pot-1:], MP, POTInfo):
 309                                         # Trimmed even transactions with fees
 310                                         pass
 311                                 else:
 312                                         pot = idealtxncount
 313                                         self._floodWarning(now, 'Non-POT', doin='Making merkle tree with %d transactions (ideal: %d; max: %d)' % (pot, idealtxncount, txncount))
 314                         POTInfo[1][1] = pot
 315                         pot -= 1
 316                         txnlist[pot:] = ()
 317                         txninfo[pot:] = ()
 318
 319         def _CallGBT(self, TS):
 320                 access = TS['access']
 321                 self.logger.debug('Requesting new template from \'%s\'' % (TS['name'],))
 322                 try:
 323                         # First, try BIP 22 standard getblocktemplate :)
 324                         MP = access.getblocktemplate(self.GBTReq)
 325                         access.OldGMP = False
 326                 except:
 327                         try:
 328                                 # Failing that, give BIP 22 draft (2012-02 through 2012-07) getmemorypool a chance
 329                                 MP = access.getmemorypool(self.GMPReq)
 330                         except:
 331                                 try:
 332                                         # Finally, fall back to bitcoind 0.5/0.6 getmemorypool
 333                                         MP = access.getmemorypool()
 334                                 except:
 335                                         MP = False
 336                         if MP is False:
 337                                 # This way, we get the error from the BIP22 call if the old one fails too
 338                                 raise
 339
 340                         # Pre-BIP22 server (bitcoind <0.7 or Eloipool <20120513)
 341                         if not access.OldGMP:
 342                                 access.OldGMP = True
 343                                 self.logger.warning('Upstream \'%s\' is not BIP 22 compatible' % (TS['name'],))
 344
 345                 return MP
 346
 347         def _ProcessGBT(self, MP, TS = None):
 348                 oMP = MP
 349                 MP = deepcopy(MP)
 350
 351                 prevBlock = bytes.fromhex(MP['previousblockhash'])[::-1]
 352                 if 'height' not in MP:
 353                         MP['height'] = TS['access'].getinfo()['blocks'] + 1
 354                 height = MP['height']
 355                 bits = bytes.fromhex(MP['bits'])[::-1]
 356                 (MP['_bits'], MP['_prevBlock']) = (bits, prevBlock)
 357                 if (prevBlock, height, bits) != self.currentBlock and (self.currentBlock[1] is None or height > self.currentBlock[1]):
 358                         self.updateBlock(prevBlock, height, bits, _HBH=(MP['previousblockhash'], MP['bits']))
 359
 360                 txnlist = MP['transactions']
 361                 if len(txnlist) and isinstance(txnlist[0], dict):
 362                         txninfo = txnlist
 363                         txnlist = tuple(a['data'] for a in txnlist)
 364                 elif 'transactionfees' in MP:
 365                         # Backward compatibility with pre-BIP22 gmp_fees branch
 366                         txninfo = [{'fee':a} for a in MP['transactionfees']]
 367                 else:
 368                         # Backward compatibility with pre-BIP22 hex-only (bitcoind <0.7, Eloipool <future)
 369                         txninfo = [{}] * len(txnlist)
 370                 # TODO: cache Txn or at least txid from previous merkle roots?
 371                 txnlist = [a for a in map(bytes.fromhex, txnlist)]
 372
 373                 self._makeBlockSafe(MP, txnlist, txninfo)
 374
 375                 cbtxn = self.makeCoinbaseTxn(MP['coinbasevalue'])
 376                 cbtxn.setCoinbase(b'\0\0')
 377                 cbtxn.assemble()
 378                 txnlist.insert(0, cbtxn.data)
 379                 txninfo.insert(0, {
 380                 })
 381
 382                 txnlist = [a for a in map(Txn, txnlist[1:])]
 383                 txnlist.insert(0, cbtxn)
 384                 txnlist = list(txnlist)
 385                 newMerkleTree = MerkleTree(txnlist)
 386                 newMerkleTree.POTInfo = MP.get('POTInfo')
 387                 newMerkleTree.MP = MP
 388                 newMerkleTree.oMP = oMP
 389
 390                 return newMerkleTree
 391
 392         def _CheckTemplate(self, newMerkleTree, TS):
 393                 TCList = self.TemplateChecks
 394                 if not TCList:
 395                         if 'proposal' not in newMerkleTree.oMP.get('capabilities', ()):
 396                                 return (None, None)
 397                         TCList = (
 398                                 {
 399                                         'name': TS['name'],
 400                                         'access': TS['access'],
 401                                         'unanimous': True,
 402                                         'weight': 1,
 403                                 },
 404                         )
 405
 406                 MP = newMerkleTree.MP
 407                 (prevBlock, height, bits) = (MP['_prevBlock'], MP['height'], MP['_bits'])
 408                 txnlist = newMerkleTree.data
 409                 cbtxn = txnlist[0]
 410
 411                 coinbase = self.makeCoinbase(height=height)
 412                 cbtxn.setCoinbase(coinbase)
 413                 cbtxn.assemble()
 414                 merkleRoot = newMerkleTree.merkleRoot()
 415                 MRD = (merkleRoot, newMerkleTree, coinbase, prevBlock, bits)
 416                 blkhdr = MakeBlockHeader(MRD)
 417                 data = assembleBlock(blkhdr, txnlist)
 418                 ProposeReq = {
 419                         "mode": "proposal",
 420                         "data": b2a_hex(data).decode('utf8'),
 421                 }
 422
 423                 AcceptedScore = 0
 424                 RejectedScore = 0
 425                 Rejections = {}
 426                 ProposalErrors = {}
 427                 for TC in TCList:
 428                         caccess = TC['access']
 429                         try:
 430                                 propose = caccess.getblocktemplate(ProposeReq)
 431                         except (socket.error, ValueError) as e:
 432                                 self.logger.error('Upstream \'%s\' errored on proposal from \'%s\': %s' % (TC['name'], TS['name'], e))
 433                                 ProposalErrors[TC['name']] = e
 434                                 continue
 435                         if propose is None:
 436                                 AcceptedScore += TC['weight']
 437                                 self.logger.debug('Upstream \'%s\' accepted proposal' % (TC['name'],))
 438                         elif propose == 'orphan':
 439                                 self.logger.debug('Upstream \'%s\' considered proposal an orphan' % (TC['name'],))
 440                                 ProposalErrors[TC['name']] = propose
 441                         else:
 442                                 RejectedScore += TC['weight']
 443                                 Rejections[TC['name']] = propose
 444                                 try:
 445                                         propose = propose['reject-reason']
 446                                 except:
 447                                         pass
 448                                 self.logger.error('Upstream \'%s\' rejected proposed block from \'%s\': %s' % (TC['name'], TS['name'], propose))
 449
 450                 if Rejections:
 451                         RPInfo = {
 452                                 'merkleTree': newMerkleTree,
 453                                 'AcceptedScore': AcceptedScore,
 454                                 'RejectedScore': RejectedScore,
 455                                 'Rejections': Rejections,
 456                                 'ProposalErrors': ProposalErrors,
 457                         }
 458                         self.RejectedProposal = RPInfo
 459
 460                         try:
 461                                 global _filecounter
 462                                 _filecounter += 1
 463                                 import pickle
 464                                 with open('RejectedProposals/%d_%d' % (int(time()), _filecounter), 'wb') as f:
 465                                         pickle.dump(RPInfo, f)
 466                         except IOError:
 467                                 pass
 468
 469                 TotalScore = AcceptedScore + RejectedScore
 470
 471                 return (AcceptedScore, TotalScore)
 472
 473         def _updateMerkleTree_fromTS(self, TS):
 474                 MP = self._CallGBT(TS)
 475                 newMerkleTree = self._ProcessGBT(MP, TS)
 476
 477                 # Some versions of bitcoinrpc ServiceProxy have problems copying/pickling, so just store name and URI for now
 478                 newMerkleTree.source = TS['name']
 479                 newMerkleTree.source_uri = TS['uri']
 480
 481                 (AcceptedScore, TotalScore) = self._CheckTemplate(newMerkleTree, TS)
 482                 if TotalScore is None:
 483                         return (0, newMerkleTree)
 484
 485                 AcceptRatio = AcceptedScore / TotalScore
 486
 487                 self.logger.debug('Template from \'%s\' has %s acceptance ratio and score of %s' % (TS['name'], AcceptRatio, AcceptedScore))
 488
 489                 if AcceptRatio <= self.MinimumTemplateAcceptanceRatio:
 490                         return None
 491
 492                 if TotalScore < self.MinimumTemplateScore:
 493                         return None
 494
 495                 return (AcceptRatio, newMerkleTree)
 496
 497         def _updateMerkleTree_I(self):
 498                 Best = (-1, None)
 499                 for TSPriList in self.TemplateSources:
 500                         # FIXME: Implement weighting
 501                         for i in range(len(TSPriList)):
 502                                 TS = TSPriList.pop(0)
 503                                 TSPriList.append(TS)
 504
 505                                 try:
 506                                         r = self._updateMerkleTree_fromTS(TS)
 507                                         if r is None:
 508                                                 # Failed completely
 509                                                 continue
 510                                         if Best[0] < r[0]:
 511                                                 Best = r
 512                                                 if r[0] == 1:
 513                                                         break
 514                                 except:
 515                                         if TSPriList == self.TemplateSources[-1] and i == len(TSPriList) - 1 and Best[1] is None:
 516                                                 raise
 517                                         else:
 518                                                 self.logger.error(traceback.format_exc())
 519
 520                 BestMT = Best[1]
 521                 if BestMT is None:
 522                         raise RuntimeError('Failed to create usable template')
 523
 524                 self.logger.debug('Updating merkle tree with template from \'%s\'' % (BestMT.source,))
 525                 MP = BestMT.MP
 526                 blkbasics = (MP['_prevBlock'], MP['height'], MP['_bits'])
 527                 if blkbasics != self.currentBlock:
 528                         self.updateBlock(*blkbasics, _HBH=(MP['previousblockhash'], MP['bits']))
 529                 self.currentMerkleTree = BestMT
 530
 531         def updateMerkleTree(self):
 532                 global now
 533                 self.logger.debug('Polling for new block template')
 534                 self.nextMerkleUpdate = now + self.TxnUpdateRetryWait
 535
 536                 self._updateMerkleTree_I()
 537
 538                 self.lastMerkleUpdate = now
 539                 self.nextMerkleUpdate = now + self.MinimumTxnUpdateWait
 540
 541                 if self.needMerkle == 2:
 542                         self.needMerkle = 1
 543                         self.needMerkleSince = now
 544
 545         def makeCoinbase(self, height):
 546                 now = int(time())
 547                 if now > _makeCoinbase[0]:
 548                         _makeCoinbase[0] = now
 549                         _makeCoinbase[1] = 0
 550                 else:
 551                         _makeCoinbase[1] += 1
 552                 rv = self.CoinbasePrefix
 553                 rv += pack('>L', now) + pack('>Q', _makeCoinbase[1]).lstrip(b'\0')
 554                 # NOTE: Not using varlenEncode, since this is always guaranteed to be < 100
 555                 rv = bytes( (len(rv),) ) + rv
 556                 for v in self.CoinbaseAux.values():
 557                         rv += v
 558                 if len(rv) > 95:
 559                         t = time()
 560                         if self.overflowed < t - 300:
 561                                 self.logger.warning('Overflowing coinbase data! %d bytes long' % (len(rv),))
 562                                 self.overflowed = t
 563                                 self.isOverflowed = True
 564                         rv = rv[:95]
 565                 else:
 566                         self.isOverflowed = False
 567                 rv = bitcoin.script.encodeUNum(height) + rv
 568                 return rv
 569
 570         def makeMerkleRoot(self, merkleTree, height):
 571                 cbtxn = merkleTree.data[0]
 572                 cb = self.makeCoinbase(height=height)
 573                 cbtxn.setCoinbase(cb)
 574                 cbtxn.assemble()
 575                 merkleRoot = merkleTree.merkleRoot()
 576                 return (merkleRoot, merkleTree, cb)
 577
 578         _doing_last = None
 579         def _doing(self, what):
 580                 if self._doing_last == what:
 581                         self._doing_i += 1
 582                         return
 583                 global now
 584                 if self._doing_last:
 585                         self.logger.debug("Switching from (%4dx in %5.3f seconds) %s => %s" % (self._doing_i, now - self._doing_s, self._doing_last, what))
 586                 self._doing_last = what
 587                 self._doing_i = 1
 588                 self._doing_s = now
 589
 590         def _floodWarning(self, now, wid, wmsgf = None, doin = True, logf = None):
 591                 if doin is True:
 592                         doin = self._doing_last
 593                         def a(f = wmsgf):
 594                                 return lambda: "%s (doing %s)" % (f(), doin)
 595                         wmsgf = a()
 596                 winfo = self.lastWarning.setdefault(wid, [0, None])
 597                 (lastTime, lastDoing) = winfo
 598                 if now <= lastTime + max(5, self.MinimumTxnUpdateWait):
 599                         return
 600                 winfo[0] = now
 601                 nowDoing = doin
 602                 winfo[1] = nowDoing
 603                 if logf is None:
 604                         logf = self.logger.warning
 605                 logf(wmsgf() if wmsgf else doin)
 606
 607         def _makeOne(self, putf, merkleTree, height):
 608                 MakingAtThisHeight = self.currentBlock[1]
 609                 MR = self.makeMerkleRoot(merkleTree, height=height)
 610                 # Only add it if the height hasn't changed in the meantime, to avoid a race
 611                 if self.currentBlock[1] == MakingAtThisHeight:
 612                         putf(MR)
 613
 614         def makeClear(self):
 615                 self._doing('clear merkle roots')
 616                 self._makeOne(self.clearMerkleRoots.put, self.curClearMerkleTree, height=self.currentBlock[1])
 617
 618         def makeNext(self):
 619                 self._doing('longpoll merkle roots')
 620                 self._makeOne(self.nextMerkleRoots.put, self.nextMerkleTree, height=self.currentBlock[1] + 1)
 621
 622         def makeRegular(self):
 623                 self._doing('regular merkle roots')
 624                 self._makeOne(self.merkleRoots.append, self.currentMerkleTree, height=self.currentBlock[1])
 625
 626         def merkleMaker_II(self):
 627                 global now
 628
 629                 # No bits = no mining :(
 630                 if not self.ready:
 631                         return self.updateMerkleTree()
 632
 633                 # First, ensure we have the minimum clear, next, and regular (in that order)
 634                 if self.clearMerkleRoots.qsize() < self.WorkQueueSizeClear[0]:
 635                         return self.makeClear()
 636                 if self.nextMerkleRoots.qsize() < self.WorkQueueSizeLongpoll[0]:
 637                         return self.makeNext()
 638                 if len(self.merkleRoots) < self.WorkQueueSizeRegular[0]:
 639                         return self.makeRegular()
 640
 641                 # If we've met the minimum requirements, consider updating the merkle tree
 642                 if self.nextMerkleUpdate <= now:
 643                         return self.updateMerkleTree()
 644
 645                 # Finally, fill up clear, next, and regular until we've met the maximums
 646                 if self.clearMerkleRoots.qsize() < self.WorkQueueSizeClear[1]:
 647                         return self.makeClear()
 648                 if self.nextMerkleRoots.qsize() < self.WorkQueueSizeLongpoll[1]:
 649                         return self.makeNext()
 650                 if len(self.merkleRoots) < self.WorkQueueSizeRegular[1] or self.merkleRoots[0][1] != self.currentMerkleTree:
 651                         return self.makeRegular()
 652
 653                 # Nothing left to do, fire onBlockUpdate event (if appropriate) and sleep
 654                 if self.needMerkle == 1:
 655                         self.onBlockUpdate()
 656                         self.needMerkle = False
 657                 self._doing('idle')
 658                 # TODO: rather than sleepspin, block until MinimumTxnUpdateWait expires or threading.Condition(?)
 659                 sleep(self.IdleSleepTime)
 660
 661         def merkleMaker_I(self):
 662                 global now
 663                 now = time()
 664
 665                 self.merkleMaker_II()
 666
 667                 if self.needMerkle == 1 and now > self.needMerkleSince + self.WarningDelayTxnLongpoll:
 668                         self._floodWarning(now, 'NeedMerkle', lambda: 'Transaction-longpoll requested %d seconds ago, and still not ready. Is your server fast enough to keep up with your configured WorkQueueSizeRegular maximum?' % (now - self.needMerkleSince,))
 669                 if now > self.nextMerkleUpdate + self.WarningDelayMerkleUpdate:
 670                         self._floodWarning(now, 'MerkleUpdate', lambda: "Haven't updated the merkle tree in at least %d seconds! Is your server fast enough to keep up with your configured work queue minimums?" % (now - self.lastMerkleUpdate,))
 671
 672         def run(self):
 673                 while True:
 674                         try:
 675                                 self.merkleMaker_I()
 676                         except:
 677                                 self.logger.critical(traceback.format_exc())
 678
 679         def start(self, *a, **k):
 680                 self._prepare()
 681                 super().start(*a, **k)
 682
 683         def getMRD(self):
 684                 try:
 685                         MRD = self.merkleRoots.pop()
 686                         self.LowestMerkleRoots = min(len(self.merkleRoots), self.LowestMerkleRoots)
 687                         rollPrevBlk = False
 688                 except IndexError:
 689                         qsz = self.clearMerkleRoots.qsize()
 690                         if qsz < 0x10:
 691                                 self.logger.warning('clearMerkleRoots running out! only %d left' % (qsz,))
 692                         MRD = None
 693                         while MRD is None:
 694                                 MRD = self.clearMerkleRoots.get()
 695                         self.LowestClearMerkleRoots = min(self.clearMerkleRoots.qsize(), self.LowestClearMerkleRoots)
 696                         rollPrevBlk = True
 697                 (merkleRoot, merkleTree, cb) = MRD
 698                 (prevBlock, height, bits) = self.currentBlock
 699                 return (merkleRoot, merkleTree, cb, prevBlock, bits, rollPrevBlk)
 700
 701         def getMC(self, wantClear = False):
 702                 if not self.ready:
 703                         with self.readyCV:
 704                                 while not self.ready:
 705                                         self.readyCV.wait()
 706                 (prevBlock, height, bits) = self.currentBlock
 707                 mt = self.curClearMerkleTree if wantClear else self.currentMerkleTree
 708                 cb = self.makeCoinbase(height=height)
 709                 rollPrevBlk = (mt == self.curClearMerkleTree)
 710                 return (height, mt, cb, prevBlock, bits, rollPrevBlk)
 711
 712 # merkleMaker tests
 713 def _test():
 714         global now
 715         now = 1337039788
 716         MM = merkleMaker()
 717         reallogger = MM.logger
 718         class fakelogger:
 719                 LO = False
 720                 def critical(self, *a):
 721                         if self.LO > 1: return
 722                         reallogger.critical(*a)
 723                 def warning(self, *a):
 724                         if self.LO: return
 725                         reallogger.warning(*a)
 726                 def debug(self, *a):
 727                         pass
 728         MM.logger = fakelogger()
 729         class NMTClass:
 730                 pass
 731
 732         # _makeBlockSafe tests
 733         from copy import deepcopy
 734         MP = {
 735                 'coinbasevalue':50,
 736         }
 737         txnlist = [b'\0', b'\x01', b'\x02']
 738         txninfo = [{'fee':0, 'sigops':1}, {'fee':5, 'sigops':10000}, {'fee':0, 'sigops':10001}]
 739         def MBS(LO = 0):
 740                 m = deepcopy( (MP, txnlist, txninfo) )
 741                 MM.logger.LO = LO
 742                 try:
 743                         MM._makeBlockSafe(*m)
 744                 except:
 745                         if LO < 2:
 746                                 raise
 747                 else:
 748                         assert LO < 2  # An expected error wasn't thrown
 749                 if 'POTInfo' in m[0]:
 750                         del m[0]['POTInfo']
 751                 return m
 752         MM.POT = 0
 753         assert MBS() == (MP, txnlist[:2], txninfo[:2])
 754         txninfo[2]['fee'] = 1
 755         MPx = deepcopy(MP)
 756         MPx['coinbasevalue'] -= 1
 757         assert MBS() == (MPx, txnlist[:2], txninfo[:2])
 758         txninfo[2]['sigops'] = 1
 759         assert MBS(1) == (MP, txnlist, txninfo)
 760         # APOT tests
 761         MM.POT = 2
 762         txnlist.append(b'\x03')
 763         txninfo.append({'fee':1, 'sigops':0})
 764         MPx = deepcopy(MP)
 765         MPx['coinbasevalue'] -= 1
 766         assert MBS() == (MPx, txnlist[:3], txninfo[:3])
 767         # POT tests
 768         MM.POT = 1
 769         MM.Greedy = True
 770         txninfo[1]['fee'] = 0
 771         txninfo[2]['fee'] = 0
 772         assert MBS(1) == (MP, txnlist, txninfo)
 773         # _ProcessGBT tests
 774         def makeCoinbaseTxn(coinbaseValue, useCoinbaser = True):
 775                 txn = Txn.new()
 776                 txn.addOutput(coinbaseValue, b'')
 777                 return txn
 778         MM.makeCoinbaseTxn = makeCoinbaseTxn
 779         MM.updateBlock = lambda *a, **ka: None
 780         gbt = {
 781                 'transactions': [
 782                         {'data': '11', 'depends': [], 'fee': 1, 'sigops': 1},
 783                         {'data': '11', 'depends': [], 'fee': 0, 'sigops': 1},
 784                         {'data': '11', 'depends': [], 'fee': 0, 'sigops': 1},
 785                         {'data': '11', 'depends': [], 'fee': 1, 'sigops': 2}
 786                 ],
 787                 'height': 219507,
 788                 'coinbasevalue': 3,
 789                 'previousblockhash': '000000000000012806bc100006dc83220bd9c2ac2709dc14a0d0fa1d6f9b733c',
 790                 'version': 1,
 791                 'bits': '1a05a6b1'
 792         }
 793         nMT = MM._ProcessGBT(gbt)
 794         assert len(nMT.data) == 5
 795         nMT.data[0].disassemble()
 796         assert sum(outp[0] for outp in nMT.data[0].outputs) == 3
 797         MM.POT = 2
 798         nMT = MM._ProcessGBT(gbt)
 799         assert len(nMT.data) in (2, 4)
 800         nMT.data[0].disassemble()
 801         assert sum(outp[0] for outp in nMT.data[0].outputs) == 2
 802
 803 _test()