merklemaker.py

   1 # Eloipool - Python Bitcoin pool server
   2 # Copyright (C) 2011-2013  Luke Dashjr <luke-jr+eloipool@utopios.org>
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU Affero General Public License as
   6 # published by the Free Software Foundation, either version 3 of the
   7 # License, or (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU Affero General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU Affero General Public License
  15 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 from binascii import b2a_hex
  18 import bitcoin.script
  19 from bitcoin.script import countSigOps
  20 from bitcoin.txn import Txn
  21 from bitcoin.varlen import varlenEncode, varlenDecode
  22 from collections import deque
  23 from copy import deepcopy
  24 from queue import Queue
  25 import jsonrpc
  26 import logging
  27 from math import log
  28 from merkletree import MerkleTree
  29 import socket
  30 from struct import pack
  31 import threading
  32 from time import sleep, time
  33 import traceback
  34
  35 _makeCoinbase = [0, 0]
  36 _filecounter = 0
  37
  38 def MakeBlockHeader(MRD):
  39         (merkleRoot, merkleTree, coinbase, prevBlock, bits) = MRD[:5]
  40         timestamp = pack('<L', int(time()))
  41         hdr = b'\2\0\0\0' + prevBlock + merkleRoot + timestamp + bits + b'iolE'
  42         return hdr
  43
  44 def assembleBlock(blkhdr, txlist):
  45         payload = blkhdr
  46         payload += varlenEncode(len(txlist))
  47         for tx in txlist:
  48                 payload += tx.data
  49         return payload
  50
  51 class merkleMaker(threading.Thread):
  52         GBTCaps = [
  53                 'coinbasevalue',
  54                 'coinbase/append',
  55                 'coinbase',
  56                 'generation',
  57                 'time',
  58                 'transactions/remove',
  59                 'prevblock',
  60         ]
  61         GBTReq = {
  62                 'capabilities': GBTCaps,
  63         }
  64         GMPReq = {
  65                 'capabilities': GBTCaps,
  66                 'tx': 'obj',
  67         }
  68
  69         def __init__(self, *a, **k):
  70                 super().__init__(*a, **k)
  71                 self.daemon = True
  72                 self.logger = logging.getLogger('merkleMaker')
  73                 self.CoinbasePrefix = b''
  74                 self.CoinbaseAux = {}
  75                 self.isOverflowed = False
  76                 self.lastWarning = {}
  77                 self.MinimumTxnUpdateWait = 5
  78                 self.overflowed = 0
  79                 self.DifficultyChangeMod = 2016
  80                 self.MinimumTemplateAcceptanceRatio = 0
  81                 self.MinimumTemplateScore = 1
  82                 self.currentBlock = (None, None, None)
  83                 self.lastBlock = (None, None, None)
  84                 self.SubsidyAlgo = lambda height: 5000000000 >> (height // 210000)
  85
  86         def _prepare(self):
  87                 self.TemplateSources = list(getattr(self, 'TemplateSources', ()))
  88                 self.TemplateChecks = list(getattr(self, 'TemplateChecks', ()))
  89                 if getattr(self, 'BlockSubmissions', None) is None:
  90                         self.BlockSubmissions = ()
  91                 if hasattr(self, 'UpstreamURI'):
  92                         self.TemplateSources.append({
  93                                 'name': 'UpstreamURI',
  94                                 'uri': self.UpstreamURI,
  95                         })
  96                 URI2Name = {}
  97                 Name2URI = {}
  98                 for a in (self.TemplateSources + self.TemplateChecks + list(self.BlockSubmissions)):
  99                         if not ('name' in a and 'uri' in a):
 100                                 continue
 101                         URI2Name.setdefault(a['uri'], a['name'])
 102                         Name2URI.setdefault(a['name'], a['uri'])
 103                 def URINamePair(a, defname):
 104                         if 'name' not in a:
 105                                 a['name'] = URI2Name.get(a['uri'], defname)
 106                         elif 'uri' not in a:
 107                                 a['uri'] = Name2URI[a['name']]
 108                 _URI2Access = {}
 109                 def URI2Access(uri):
 110                         if uri not in _URI2Access:
 111                                 access = jsonrpc.ServiceProxy(uri)
 112                                 access.OldGMP = False
 113                                 _URI2Access[uri] = access
 114                         return _URI2Access[uri]
 115                 LeveledTS = {}
 116                 for i in range(len(self.TemplateSources)):
 117                         TS = self.TemplateSources[i]
 118                         URINamePair(TS, 'TemplateSources[%u]' % (i,))
 119                         TS.setdefault('priority', 0)
 120                         TS.setdefault('weight', 1)
 121                         TS['access'] = URI2Access(TS['uri'])
 122                         LeveledTS.setdefault(TS['priority'], []).append(TS)
 123                 LeveledTS = tuple(x[1] for x in sorted(LeveledTS.items()))
 124                 self.TemplateSources = LeveledTS
 125                 for i in range(len(self.TemplateChecks)):
 126                         TC = self.TemplateChecks[i]
 127                         URINamePair(TC, 'TemplateChecks[%u]' % (i,))
 128                         TC.setdefault('unanimous', False)
 129                         TC.setdefault('weight', 1)
 130                         TC['access'] = URI2Access(TC['uri'])
 131                 for i in range(len(getattr(self, 'BlockSubmissions', ()))):
 132                         BS = self.BlockSubmissions[i]
 133                         URINamePair(BS, 'BlockSubmissions[%u]' % (i,))
 134                         BS['access'] = URI2Access(BS['uri'])
 135
 136                 self.ready = False
 137                 self.readyCV = threading.Condition()
 138
 139                 self.currentMerkleTree = None
 140                 self.merkleRoots = deque(maxlen=self.WorkQueueSizeRegular[1])
 141                 self.LowestMerkleRoots = self.WorkQueueSizeRegular[1]
 142
 143                 if not hasattr(self, 'WorkQueueSizeClear'):
 144                         self.WorkQueueSizeClear = self.WorkQueueSizeLongpoll
 145                 self._MaxClearSize = max(self.WorkQueueSizeClear[1], self.WorkQueueSizeLongpoll[1])
 146                 self.clearMerkleRoots = Queue(self._MaxClearSize)
 147                 self.LowestClearMerkleRoots = self.WorkQueueSizeClear[1]
 148                 self.nextMerkleRoots = Queue(self._MaxClearSize)
 149
 150                 if not hasattr(self, 'WarningDelay'):
 151                         self.WarningDelay = max(15, self.MinimumTxnUpdateWait * 2)
 152                 if not hasattr(self, 'WarningDelayTxnLongpoll'):
 153                         self.WarningDelayTxnLongpoll = self.WarningDelay
 154                 if not hasattr(self, 'WarningDelayMerkleUpdate'):
 155                         self.WarningDelayMerkleUpdate = self.WarningDelay
 156
 157                 self.lastMerkleUpdate = 0
 158                 self.nextMerkleUpdate = 0
 159
 160         def createClearMerkleTree(self, height):
 161                 subsidy = self.SubsidyAlgo(height)
 162                 cbtxn = self.makeCoinbaseTxn(subsidy, False)
 163                 cbtxn.assemble()
 164                 return MerkleTree([cbtxn])
 165
 166         def updateBlock(self, newBlock, height = None, bits = None, _HBH = None):
 167                 if newBlock == self.currentBlock[0]:
 168                         if height in (None, self.currentBlock[1]) and bits in (None, self.currentBlock[2]):
 169                                 return
 170                         if not self.currentBlock[2] is None:
 171                                 self.logger.error('Was working on block with wrong specs: %s (height: %d->%d; bits: %s->%s' % (
 172                                         b2a_hex(newBlock[::-1]).decode('utf8'),
 173                                         self.currentBlock[1],
 174                                         height,
 175                                         b2a_hex(self.currentBlock[2][::-1]).decode('utf8'),
 176                                         b2a_hex(bits[::-1]).decode('utf8'),
 177                                 ))
 178                                 if self.needMerkle == 1:
 179                                         self.needMerkle = False
 180                                 self.onBlockUpdate()
 181
 182                 # Old block is invalid
 183                 if self.currentBlock[0] != newBlock:
 184                         self.lastBlock = self.currentBlock
 185
 186                 lastHeight = self.currentBlock[1]
 187                 if height is None:
 188                         height = self.currentBlock[1] + 1
 189                 if bits is None:
 190                         if height % self.DifficultyChangeMod == 1 or self.currentBlock[2] is None:
 191                                 self.logger.warning('New block: %s (height %d; bits: UNKNOWN)' % (b2a_hex(newBlock[::-1]).decode('utf8'), height))
 192                                 # Pretend to be 1 lower height, so we possibly retain nextMerkleRoots
 193                                 self.currentBlock = (None, height - 1, None)
 194                                 OCMR = self.clearMerkleRoots
 195                                 self.clearMerkleRoots = Queue(self.WorkQueueSizeClear[1])
 196                                 if OCMR.empty():
 197                                         OCMR.put(None)
 198                                 self.merkleRoots.clear()
 199                                 self.ready = False
 200                                 return
 201                         else:
 202                                 bits = self.currentBlock[2]
 203
 204                 if _HBH is None:
 205                         _HBH = (b2a_hex(newBlock[::-1]).decode('utf8'), b2a_hex(bits[::-1]).decode('utf8'))
 206                 self.logger.info('New block: %s (height: %d; bits: %s)' % (_HBH[0], height, _HBH[1]))
 207                 self.currentBlock = (newBlock, height, bits)
 208
 209                 if lastHeight != height:
 210                         # TODO: Perhaps reuse clear merkle trees more intelligently
 211                         OCMR = self.clearMerkleRoots
 212                         if lastHeight == height - 1:
 213                                 self.curClearMerkleTree = self.nextMerkleTree
 214                                 self.clearMerkleRoots = self.nextMerkleRoots
 215                                 self.logger.debug('Adopting next-height clear merkleroots :)')
 216                         else:
 217                                 if lastHeight:
 218                                         self.logger.warning('Change from height %d->%d; no longpoll merkleroots available!' % (lastHeight, height))
 219                                 self.curClearMerkleTree = self.createClearMerkleTree(height)
 220                                 self.clearMerkleRoots = Queue(self.WorkQueueSizeClear[1])
 221                         if OCMR.empty():
 222                                 OCMR.put(None)
 223                         self.nextMerkleTree = self.createClearMerkleTree(height + 1)
 224                         self.nextMerkleRoots = Queue(self._MaxClearSize)
 225                 else:
 226                         self.logger.debug('Already using clear merkleroots for this height')
 227                 self.currentMerkleTree = self.curClearMerkleTree
 228                 self.merkleRoots.clear()
 229
 230                 if not self.ready:
 231                         self.ready = True
 232                         with self.readyCV:
 233                                 self.readyCV.notify_all()
 234
 235                 self.needMerkle = 2
 236                 self.onBlockChange()
 237
 238         def _trimBlock(self, MP, txnlist, txninfo, floodn, msgf):
 239                 fee = txninfo[-1].get('fee', None)
 240                 if fee is None:
 241                         raise self._floodCritical(now, floodn, doin=msgf('fees unknown'))
 242                 if fee:
 243                         # FIXME: coinbasevalue is *not* guaranteed to exist here
 244                         MP['coinbasevalue'] -= fee
 245
 246                 txnlist[-1:] = ()
 247                 txninfo[-1:] = ()
 248
 249                 return True
 250
 251         # Aggressive "Power Of Two": Remove transactions even with fees to reach our goal
 252         def _APOT(self, txninfopot, MP, POTInfo):
 253                 feeTxnsTrimmed = 0
 254                 feesTrimmed = 0
 255                 for txn in txninfopot:
 256                         if txn.get('fee') is None:
 257                                 self._floodWarning(now, 'APOT-No-Fees', doin='Upstream didn\'t provide fee information required for aggressive POT', logf=self.logger.info)
 258                                 return
 259                         if not txn['fee']:
 260                                 continue
 261                         feesTrimmed += txn['fee']
 262                         feeTxnsTrimmed += 1
 263                 MP['coinbasevalue'] -= feesTrimmed
 264
 265                 POTInfo[2] = [feeTxnsTrimmed, feesTrimmed]
 266                 self._floodWarning(now, 'POT-Trimming-Fees', doin='Aggressive POT trimming %d transactions with %d.%08d BTC total fees' % (feeTxnsTrimmed, feesTrimmed//100000000, feesTrimmed % 100000000), logf=self.logger.debug)
 267
 268                 return True
 269
 270         def _makeBlockSafe(self, MP, txnlist, txninfo):
 271                 blocksize = sum(map(len, txnlist)) + 80
 272                 while blocksize > 934464:  # 1 "MB" limit - 64 KB breathing room
 273                         txnsize = len(txnlist[-1])
 274                         self._trimBlock(MP, txnlist, txninfo, 'SizeLimit', lambda x: 'Making blocks over 1 MB size limit (%d bytes; %s)' % (blocksize, x))
 275                         blocksize -= txnsize
 276
 277                 # NOTE: This check doesn't work at all without BIP22 transaction obj format
 278                 blocksigops = sum(a.get('sigops', 0) for a in txninfo)
 279                 while blocksigops > 19488:  # 20k limit - 0x200 breathing room
 280                         txnsigops = txninfo[-1]['sigops']
 281                         self._trimBlock(MP, txnlist, txninfo, 'SigOpLimit', lambda x: 'Making blocks over 20k SigOp limit (%d; %s)' % (blocksigops, x))
 282                         blocksigops -= txnsigops
 283
 284                 # Aim to produce blocks with "Power Of Two" transaction counts
 285                 # This helps avoid any chance of someone abusing CVE-2012-2459 with them
 286                 POTMode = getattr(self, 'POT', 1)
 287                 txncount = len(txnlist) + 1
 288                 if POTMode:
 289                         feetxncount = txncount
 290                         for i in range(txncount - 2, -1, -1):
 291                                 if 'fee' not in txninfo[i] or txninfo[i]['fee']:
 292                                         break
 293                                 feetxncount -= 1
 294
 295                         if getattr(self, 'Greedy', None):
 296                                 # Aim to cut off extra zero-fee transactions on the end
 297                                 # NOTE: not cutting out ones intermixed, in case of dependencies
 298                                 idealtxncount = feetxncount
 299                         else:
 300                                 idealtxncount = txncount
 301
 302                         pot = 2**int(log(idealtxncount, 2))
 303                         POTInfo = MP['POTInfo'] = [[idealtxncount, feetxncount, txncount], [pot, None], None]
 304                         if pot < idealtxncount:
 305                                 if pot * 2 <= txncount:
 306                                         pot *= 2
 307                                 elif pot >= feetxncount:
 308                                         pass
 309                                 elif POTMode > 1:
 310                                         if self._APOT(txninfo[pot-1:], MP, POTInfo):
 311                                                 # Trimmed even transactions with fees
 312                                                 pass
 313                                         else:
 314                                                 pot = idealtxncount
 315                                                 self._floodWarning(now, 'Non-POT', doin='Making merkle tree with %d transactions (ideal: %d; max: %d)' % (pot, idealtxncount, txncount))
 316                                 else:
 317                                         pot = idealtxncount
 318                         POTInfo[1][1] = pot
 319                         pot -= 1
 320                         txnlist[pot:] = ()
 321                         txninfo[pot:] = ()
 322
 323         def _CallGBT(self, TS):
 324                 access = TS['access']
 325                 self.logger.debug('Requesting new template from \'%s\'' % (TS['name'],))
 326                 try:
 327                         # First, try BIP 22 standard getblocktemplate :)
 328                         MP = access.getblocktemplate(self.GBTReq)
 329                         access.OldGMP = False
 330                 except:
 331                         try:
 332                                 # Failing that, give BIP 22 draft (2012-02 through 2012-07) getmemorypool a chance
 333                                 MP = access.getmemorypool(self.GMPReq)
 334                         except:
 335                                 try:
 336                                         # Finally, fall back to bitcoind 0.5/0.6 getmemorypool
 337                                         MP = access.getmemorypool()
 338                                 except:
 339                                         MP = False
 340                         if MP is False:
 341                                 # This way, we get the error from the BIP22 call if the old one fails too
 342                                 raise
 343
 344                         # Pre-BIP22 server (bitcoind <0.7 or Eloipool <20120513)
 345                         if not access.OldGMP:
 346                                 access.OldGMP = True
 347                                 self.logger.warning('Upstream \'%s\' is not BIP 22 compatible' % (TS['name'],))
 348
 349                 return MP
 350
 351         def _ProcessGBT(self, MP, TS = None):
 352                 oMP = MP
 353                 MP = deepcopy(MP)
 354
 355                 prevBlock = bytes.fromhex(MP['previousblockhash'])[::-1]
 356                 if 'height' not in MP:
 357                         MP['height'] = TS['access'].getinfo()['blocks'] + 1
 358                 height = MP['height']
 359                 bits = bytes.fromhex(MP['bits'])[::-1]
 360                 (MP['_bits'], MP['_prevBlock']) = (bits, prevBlock)
 361                 if (prevBlock, height, bits) != self.currentBlock and (self.currentBlock[1] is None or height > self.currentBlock[1]):
 362                         self.updateBlock(prevBlock, height, bits, _HBH=(MP['previousblockhash'], MP['bits']))
 363
 364                 txnlist = MP['transactions']
 365                 if len(txnlist) and isinstance(txnlist[0], dict):
 366                         txninfo = txnlist
 367                         txnlist = tuple(a['data'] for a in txnlist)
 368                 elif 'transactionfees' in MP:
 369                         # Backward compatibility with pre-BIP22 gmp_fees branch
 370                         txninfo = [{'fee':a} for a in MP['transactionfees']]
 371                 else:
 372                         # Backward compatibility with pre-BIP22 hex-only (bitcoind <0.7, Eloipool <future)
 373                         txninfo = [{}] * len(txnlist)
 374                 # TODO: cache Txn or at least txid from previous merkle roots?
 375                 txnlist = [a for a in map(bytes.fromhex, txnlist)]
 376
 377                 self._makeBlockSafe(MP, txnlist, txninfo)
 378
 379                 cbtxn = self.makeCoinbaseTxn(MP['coinbasevalue'], prevBlockHex = MP['previousblockhash'])
 380                 cbtxn.setCoinbase(b'\0\0')
 381                 cbtxn.assemble()
 382                 txnlist.insert(0, cbtxn.data)
 383                 txninfo.insert(0, {
 384                 })
 385
 386                 txnlist = [a for a in map(Txn, txnlist[1:])]
 387                 txnlist.insert(0, cbtxn)
 388                 txnlist = list(txnlist)
 389                 newMerkleTree = MerkleTree(txnlist)
 390                 newMerkleTree.POTInfo = MP.get('POTInfo')
 391                 newMerkleTree.MP = MP
 392                 newMerkleTree.oMP = oMP
 393
 394                 return newMerkleTree
 395
 396         def _CheckTemplate(self, newMerkleTree, TS):
 397                 TCList = self.TemplateChecks
 398                 if not TCList:
 399                         if 'proposal' not in newMerkleTree.oMP.get('capabilities', ()):
 400                                 return (None, None)
 401                         TCList = (
 402                                 {
 403                                         'name': TS['name'],
 404                                         'access': TS['access'],
 405                                         'unanimous': True,
 406                                         'weight': 1,
 407                                 },
 408                         )
 409
 410                 MP = newMerkleTree.MP
 411                 (prevBlock, height, bits) = (MP['_prevBlock'], MP['height'], MP['_bits'])
 412                 txnlist = newMerkleTree.data
 413                 cbtxn = txnlist[0]
 414
 415                 coinbase = self.makeCoinbase(height=height)
 416                 cbtxn.setCoinbase(coinbase)
 417                 cbtxn.assemble()
 418                 merkleRoot = newMerkleTree.merkleRoot()
 419                 MRD = (merkleRoot, newMerkleTree, coinbase, prevBlock, bits)
 420                 blkhdr = MakeBlockHeader(MRD)
 421                 data = assembleBlock(blkhdr, txnlist)
 422                 ProposeReq = {
 423                         "mode": "proposal",
 424                         "data": b2a_hex(data).decode('utf8'),
 425                 }
 426
 427                 AcceptedScore = 0
 428                 RejectedScore = 0
 429                 Rejections = {}
 430                 ProposalErrors = {}
 431                 for TC in TCList:
 432                         caccess = TC['access']
 433                         try:
 434                                 propose = caccess.getblocktemplate(ProposeReq)
 435                         except (socket.error, ValueError) as e:
 436                                 self.logger.error('Upstream \'%s\' errored on proposal from \'%s\': %s' % (TC['name'], TS['name'], e))
 437                                 ProposalErrors[TC['name']] = e
 438                                 continue
 439                         if propose is None:
 440                                 AcceptedScore += TC['weight']
 441                                 self.logger.debug('Upstream \'%s\' accepted proposal' % (TC['name'],))
 442                         elif propose == 'orphan':
 443                                 self.logger.debug('Upstream \'%s\' considered proposal an orphan' % (TC['name'],))
 444                                 ProposalErrors[TC['name']] = propose
 445                         else:
 446                                 RejectedScore += TC['weight']
 447                                 Rejections[TC['name']] = propose
 448                                 try:
 449                                         propose = propose['reject-reason']
 450                                 except:
 451                                         pass
 452                                 self.logger.error('Upstream \'%s\' rejected proposed block from \'%s\': %s' % (TC['name'], TS['name'], propose))
 453
 454                 if Rejections:
 455                         RPInfo = {
 456                                 'merkleTree': newMerkleTree,
 457                                 'AcceptedScore': AcceptedScore,
 458                                 'RejectedScore': RejectedScore,
 459                                 'Rejections': Rejections,
 460                                 'ProposalErrors': ProposalErrors,
 461                         }
 462                         self.RejectedProposal = RPInfo
 463
 464                         try:
 465                                 global _filecounter
 466                                 _filecounter += 1
 467                                 import pickle
 468                                 with open('RejectedProposals/%d_%d' % (int(time()), _filecounter), 'wb') as f:
 469                                         pickle.dump(RPInfo, f)
 470                         except IOError:
 471                                 pass
 472
 473                 TotalScore = AcceptedScore + RejectedScore
 474
 475                 return (AcceptedScore, TotalScore)
 476
 477         def _updateMerkleTree_fromTS(self, TS):
 478                 MP = self._CallGBT(TS)
 479                 newMerkleTree = self._ProcessGBT(MP, TS)
 480
 481                 # Some versions of bitcoinrpc ServiceProxy have problems copying/pickling, so just store name and URI for now
 482                 newMerkleTree.source = TS['name']
 483                 newMerkleTree.source_uri = TS['uri']
 484
 485                 (AcceptedScore, TotalScore) = self._CheckTemplate(newMerkleTree, TS)
 486                 if TotalScore is None:
 487                         return (0, newMerkleTree)
 488
 489                 if TotalScore:
 490                         AcceptRatio = AcceptedScore / TotalScore
 491                 else:
 492                         AcceptRatio = 0.0
 493
 494                 self.logger.debug('Template from \'%s\' has %s acceptance ratio and score of %s' % (TS['name'], AcceptRatio, AcceptedScore))
 495
 496                 if AcceptRatio <= self.MinimumTemplateAcceptanceRatio:
 497                         return None
 498
 499                 if TotalScore < self.MinimumTemplateScore:
 500                         return None
 501
 502                 return (AcceptRatio, newMerkleTree)
 503
 504         def _updateMerkleTree_I(self):
 505                 Best = (-1, None)
 506                 for TSPriList in self.TemplateSources:
 507                         # FIXME: Implement weighting
 508                         for i in range(len(TSPriList)):
 509                                 TS = TSPriList.pop(0)
 510                                 TSPriList.append(TS)
 511
 512                                 try:
 513                                         r = self._updateMerkleTree_fromTS(TS)
 514                                         if r is None:
 515                                                 # Failed completely
 516                                                 continue
 517
 518                                         (AcceptRatio, newMerkleTree) = r
 519
 520                                         # NOTE: If you're going to try to remove this preference for the highest block, you need to (at least) stop _ProcessGBT from calling updateBlock whenever it sees a new high
 521                                         AcceptRatio += newMerkleTree.MP['height']
 522
 523                                         if Best[0] < AcceptRatio:
 524                                                 Best = r
 525                                                 if AcceptRatio == 1:
 526                                                         break
 527                                 except:
 528                                         if TSPriList == self.TemplateSources[-1] and i == len(TSPriList) - 1 and Best[1] is None:
 529                                                 raise
 530                                         else:
 531                                                 self.logger.error(traceback.format_exc())
 532
 533                 BestMT = Best[1]
 534                 if BestMT is None:
 535                         raise RuntimeError('Failed to create usable template')
 536
 537                 self.logger.debug('Updating merkle tree with template from \'%s\'' % (BestMT.source,))
 538                 MP = BestMT.MP
 539                 blkbasics = (MP['_prevBlock'], MP['height'], MP['_bits'])
 540                 if blkbasics != self.currentBlock:
 541                         self.updateBlock(*blkbasics, _HBH=(MP['previousblockhash'], MP['bits']))
 542                 self.currentMerkleTree = BestMT
 543
 544         def _updateMerkleTree(self):
 545                 global now
 546                 self.logger.debug('Polling for new block template')
 547                 self.nextMerkleUpdate = now + self.TxnUpdateRetryWait
 548
 549                 self._updateMerkleTree_I()
 550
 551                 self.lastMerkleUpdate = now
 552                 self.nextMerkleUpdate = now + self.MinimumTxnUpdateWait
 553
 554                 if self.needMerkle == 2:
 555                         self.needMerkle = 1
 556                         self.needMerkleSince = now
 557
 558         def updateMerkleTree(self):
 559                 global now
 560                 now = time()
 561                 self._updateMerkleTree()
 562
 563         def makeCoinbase(self, height):
 564                 now = int(time())
 565                 if now > _makeCoinbase[0]:
 566                         _makeCoinbase[0] = now
 567                         _makeCoinbase[1] = 0
 568                 else:
 569                         _makeCoinbase[1] += 1
 570                 rv = self.CoinbasePrefix
 571                 rv += pack('>L', now) + pack('>Q', _makeCoinbase[1]).lstrip(b'\0')
 572                 # NOTE: Not using varlenEncode, since this is always guaranteed to be < 100
 573                 rv = bytes( (len(rv),) ) + rv
 574                 for v in self.CoinbaseAux.values():
 575                         rv += v
 576                 if len(rv) > 95:
 577                         t = time()
 578                         if self.overflowed < t - 300:
 579                                 self.logger.warning('Overflowing coinbase data! %d bytes long' % (len(rv),))
 580                                 self.overflowed = t
 581                                 self.isOverflowed = True
 582                         rv = rv[:95]
 583                 else:
 584                         self.isOverflowed = False
 585                 rv = bitcoin.script.encodeUNum(height) + rv
 586                 return rv
 587
 588         def makeMerkleRoot(self, merkleTree, height):
 589                 cbtxn = merkleTree.data[0]
 590                 cb = self.makeCoinbase(height=height)
 591                 cbtxn.setCoinbase(cb)
 592                 cbtxn.assemble()
 593                 merkleRoot = merkleTree.merkleRoot()
 594                 return (merkleRoot, merkleTree, cb)
 595
 596         _doing_last = None
 597         def _doing(self, what):
 598                 if self._doing_last == what:
 599                         self._doing_i += 1
 600                         return
 601                 global now
 602                 if self._doing_last:
 603                         self.logger.debug("Switching from (%4dx in %5.3f seconds) %s => %s" % (self._doing_i, now - self._doing_s, self._doing_last, what))
 604                 self._doing_last = what
 605                 self._doing_i = 1
 606                 self._doing_s = now
 607
 608         def _floodWarning(self, now, wid, wmsgf = None, doin = True, logf = None):
 609                 if doin is True:
 610                         doin = self._doing_last
 611                         def a(f = wmsgf):
 612                                 return lambda: "%s (doing %s)" % (f(), doin)
 613                         wmsgf = a()
 614                 winfo = self.lastWarning.setdefault(wid, [0, None])
 615                 (lastTime, lastDoing) = winfo
 616                 if now <= lastTime + max(5, self.MinimumTxnUpdateWait):
 617                         return
 618                 winfo[0] = now
 619                 nowDoing = doin
 620                 winfo[1] = nowDoing
 621                 if logf is None:
 622                         logf = self.logger.warning
 623                 logf(wmsgf() if wmsgf else doin)
 624
 625         def _makeOne(self, putf, merkleTree, height):
 626                 MakingAtThisHeight = self.currentBlock[1]
 627                 MR = self.makeMerkleRoot(merkleTree, height=height)
 628                 # Only add it if the height hasn't changed in the meantime, to avoid a race
 629                 if self.currentBlock[1] == MakingAtThisHeight:
 630                         putf(MR)
 631
 632         def makeClear(self):
 633                 self._doing('clear merkle roots')
 634                 self._makeOne(self.clearMerkleRoots.put, self.curClearMerkleTree, height=self.currentBlock[1])
 635
 636         def makeNext(self):
 637                 self._doing('longpoll merkle roots')
 638                 self._makeOne(self.nextMerkleRoots.put, self.nextMerkleTree, height=self.currentBlock[1] + 1)
 639
 640         def makeRegular(self):
 641                 self._doing('regular merkle roots')
 642                 self._makeOne(self.merkleRoots.append, self.currentMerkleTree, height=self.currentBlock[1])
 643
 644         def merkleMaker_II(self):
 645                 global now
 646
 647                 # No bits = no mining :(
 648                 if not self.ready:
 649                         return self._updateMerkleTree()
 650
 651                 # First, ensure we have the minimum clear, next, and regular (in that order)
 652                 if self.clearMerkleRoots.qsize() < self.WorkQueueSizeClear[0]:
 653                         return self.makeClear()
 654                 if self.nextMerkleRoots.qsize() < self.WorkQueueSizeLongpoll[0]:
 655                         return self.makeNext()
 656                 if len(self.merkleRoots) < self.WorkQueueSizeRegular[0]:
 657                         return self.makeRegular()
 658
 659                 # If we've met the minimum requirements, consider updating the merkle tree
 660                 if self.nextMerkleUpdate <= now:
 661                         return self._updateMerkleTree()
 662
 663                 # Finally, fill up clear, next, and regular until we've met the maximums
 664                 if self.clearMerkleRoots.qsize() < self.WorkQueueSizeClear[1]:
 665                         return self.makeClear()
 666                 if self.nextMerkleRoots.qsize() < self.WorkQueueSizeLongpoll[1]:
 667                         return self.makeNext()
 668                 if len(self.merkleRoots) < self.WorkQueueSizeRegular[1] or self.merkleRoots[0][1] != self.currentMerkleTree:
 669                         return self.makeRegular()
 670
 671                 # Nothing left to do, fire onBlockUpdate event (if appropriate) and sleep
 672                 if self.needMerkle == 1:
 673                         self.onBlockUpdate()
 674                         self.needMerkle = False
 675                 self._doing('idle')
 676                 # TODO: rather than sleepspin, block until MinimumTxnUpdateWait expires or threading.Condition(?)
 677                 sleep(self.IdleSleepTime)
 678
 679         def merkleMaker_I(self):
 680                 global now
 681                 now = time()
 682
 683                 self.merkleMaker_II()
 684
 685                 if self.needMerkle == 1 and now > self.needMerkleSince + self.WarningDelayTxnLongpoll:
 686                         self._floodWarning(now, 'NeedMerkle', lambda: 'Transaction-longpoll requested %d seconds ago, and still not ready. Is your server fast enough to keep up with your configured WorkQueueSizeRegular maximum?' % (now - self.needMerkleSince,))
 687                 if now > self.nextMerkleUpdate + self.WarningDelayMerkleUpdate:
 688                         self._floodWarning(now, 'MerkleUpdate', lambda: "Haven't updated the merkle tree in at least %d seconds! Is your server fast enough to keep up with your configured work queue minimums?" % (now - self.lastMerkleUpdate,))
 689
 690         def run(self):
 691                 while True:
 692                         try:
 693                                 self.merkleMaker_I()
 694                         except:
 695                                 self.logger.critical(traceback.format_exc())
 696
 697         def start(self, *a, **k):
 698                 self._prepare()
 699                 super().start(*a, **k)
 700
 701         def getMRD(self):
 702                 try:
 703                         MRD = self.merkleRoots.pop()
 704                         self.LowestMerkleRoots = min(len(self.merkleRoots), self.LowestMerkleRoots)
 705                         rollPrevBlk = False
 706                 except IndexError:
 707                         qsz = self.clearMerkleRoots.qsize()
 708                         if qsz < 0x10:
 709                                 self.logger.warning('clearMerkleRoots running out! only %d left' % (qsz,))
 710                         MRD = None
 711                         while MRD is None:
 712                                 MRD = self.clearMerkleRoots.get()
 713                         self.LowestClearMerkleRoots = min(self.clearMerkleRoots.qsize(), self.LowestClearMerkleRoots)
 714                         rollPrevBlk = True
 715                 (merkleRoot, merkleTree, cb) = MRD
 716                 (prevBlock, height, bits) = self.currentBlock
 717                 return (merkleRoot, merkleTree, cb, prevBlock, bits, rollPrevBlk)
 718
 719         def getMC(self, wantClear = False):
 720                 if not self.ready:
 721                         with self.readyCV:
 722                                 while not self.ready:
 723                                         self.readyCV.wait()
 724                 (prevBlock, height, bits) = self.currentBlock
 725                 mt = self.curClearMerkleTree if wantClear else self.currentMerkleTree
 726                 cb = self.makeCoinbase(height=height)
 727                 rollPrevBlk = (mt == self.curClearMerkleTree)
 728                 return (height, mt, cb, prevBlock, bits, rollPrevBlk)
 729
 730 # merkleMaker tests
 731 def _test():
 732         global now
 733         now = 1337039788
 734         MM = merkleMaker()
 735         reallogger = MM.logger
 736         class fakelogger:
 737                 LO = False
 738                 def critical(self, *a):
 739                         if self.LO > 1: return
 740                         reallogger.critical(*a)
 741                 def warning(self, *a):
 742                         if self.LO: return
 743                         reallogger.warning(*a)
 744                 def debug(self, *a):
 745                         pass
 746         MM.logger = fakelogger()
 747         class NMTClass:
 748                 pass
 749
 750         # _makeBlockSafe tests
 751         from copy import deepcopy
 752         MP = {
 753                 'coinbasevalue':50,
 754         }
 755         txnlist = [b'\0', b'\x01', b'\x02']
 756         txninfo = [{'fee':0, 'sigops':1}, {'fee':5, 'sigops':10000}, {'fee':0, 'sigops':10001}]
 757         def MBS(LO = 0):
 758                 m = deepcopy( (MP, txnlist, txninfo) )
 759                 MM.logger.LO = LO
 760                 try:
 761                         MM._makeBlockSafe(*m)
 762                 except:
 763                         if LO < 2:
 764                                 raise
 765                 else:
 766                         assert LO < 2  # An expected error wasn't thrown
 767                 if 'POTInfo' in m[0]:
 768                         del m[0]['POTInfo']
 769                 return m
 770         MM.POT = 0
 771         assert MBS() == (MP, txnlist[:2], txninfo[:2])
 772         txninfo[2]['fee'] = 1
 773         MPx = deepcopy(MP)
 774         MPx['coinbasevalue'] -= 1
 775         assert MBS() == (MPx, txnlist[:2], txninfo[:2])
 776         txninfo[2]['sigops'] = 1
 777         assert MBS(1) == (MP, txnlist, txninfo)
 778         # APOT tests
 779         MM.POT = 2
 780         txnlist.append(b'\x03')
 781         txninfo.append({'fee':1, 'sigops':0})
 782         MPx = deepcopy(MP)
 783         MPx['coinbasevalue'] -= 1
 784         assert MBS() == (MPx, txnlist[:3], txninfo[:3])
 785         # POT tests
 786         MM.POT = 1
 787         MM.Greedy = True
 788         txninfo[1]['fee'] = 0
 789         txninfo[2]['fee'] = 0
 790         assert MBS(1) == (MP, txnlist, txninfo)
 791         # _ProcessGBT tests
 792         def makeCoinbaseTxn(coinbaseValue, useCoinbaser = True, prevBlockHex = None):
 793                 txn = Txn.new()
 794                 txn.addOutput(coinbaseValue, b'')
 795                 return txn
 796         MM.makeCoinbaseTxn = makeCoinbaseTxn
 797         MM.updateBlock = lambda *a, **ka: None
 798         gbt = {
 799                 'transactions': [
 800                         {'data': '11', 'depends': [], 'fee': 1, 'sigops': 1},
 801                         {'data': '11', 'depends': [], 'fee': 0, 'sigops': 1},
 802                         {'data': '11', 'depends': [], 'fee': 0, 'sigops': 1},
 803                         {'data': '11', 'depends': [], 'fee': 1, 'sigops': 2}
 804                 ],
 805                 'height': 219507,
 806                 'coinbasevalue': 3,
 807                 'previousblockhash': '000000000000012806bc100006dc83220bd9c2ac2709dc14a0d0fa1d6f9b733c',
 808                 'version': 1,
 809                 'bits': '1a05a6b1'
 810         }
 811         nMT = MM._ProcessGBT(gbt)
 812         assert len(nMT.data) == 5
 813         nMT.data[0].disassemble()
 814         assert sum(outp[0] for outp in nMT.data[0].outputs) == 3
 815         MM.POT = 2
 816         nMT = MM._ProcessGBT(gbt)
 817         assert len(nMT.data) in (2, 4)
 818         nMT.data[0].disassemble()
 819         assert sum(outp[0] for outp in nMT.data[0].outputs) == 2
 820
 821 _test()