1 # Eloipool - Python Bitcoin pool server
2 # Copyright (C) 2011-2012 Luke Dashjr <luke-jr+eloipool@utopios.org>
4 # This program is free software: you can redistribute it and/or modify
5 # it under the terms of the GNU Affero General Public License as
6 # published by the Free Software Foundation, either version 3 of the
7 # License, or (at your option) any later version.
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU Affero General Public License for more details.
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
17 from binascii import b2a_hex
19 from bitcoin.script import countSigOps
20 from bitcoin.txn import Txn
21 from bitcoin.varlen import varlenEncode, varlenDecode
22 from collections import deque
23 from copy import deepcopy
24 from queue import Queue
28 from merkletree import MerkleTree
29 from struct import pack
31 from time import sleep, time
34 _makeCoinbase = [0, 0]
36 def MakeBlockHeader(MRD):
37 (merkleRoot, merkleTree, coinbase, prevBlock, bits) = MRD[:5]
38 timestamp = pack('<L', int(time()))
39 hdr = b'\2\0\0\0' + prevBlock + merkleRoot + timestamp + bits + b'iolE'
42 def assembleBlock(blkhdr, txlist):
44 payload += varlenEncode(len(txlist))
49 class merkleMaker(threading.Thread):
57 'transactions/remove',
61 'capabilities': GBTCaps,
64 'capabilities': GBTCaps,
68 def __init__(self, *a, **k):
69 super().__init__(*a, **k)
71 self.logger = logging.getLogger('merkleMaker')
72 self.CoinbasePrefix = b''
74 self.isOverflowed = False
76 self.MinimumTxnUpdateWait = 5
78 self.DifficultyChangeMod = 2016
81 self.access = jsonrpc.ServiceProxy(self.UpstreamURI)
83 self.currentBlock = (None, None, None)
85 self.currentMerkleTree = None
86 self.merkleRoots = deque(maxlen=self.WorkQueueSizeRegular[1])
87 self.LowestMerkleRoots = self.WorkQueueSizeRegular[1]
89 if not hasattr(self, 'WorkQueueSizeClear'):
90 self.WorkQueueSizeClear = self.WorkQueueSizeLongpoll
91 self._MaxClearSize = max(self.WorkQueueSizeClear[1], self.WorkQueueSizeLongpoll[1])
92 self.clearMerkleTree = MerkleTree([self.clearCoinbaseTxn])
93 self.clearMerkleRoots = Queue(self._MaxClearSize)
94 self.LowestClearMerkleRoots = self.WorkQueueSizeClear[1]
95 self.nextMerkleRoots = Queue(self._MaxClearSize)
97 if not hasattr(self, 'WarningDelay'):
98 self.WarningDelay = max(15, self.MinimumTxnUpdateWait * 2)
99 if not hasattr(self, 'WarningDelayTxnLongpoll'):
100 self.WarningDelayTxnLongpoll = self.WarningDelay
101 if not hasattr(self, 'WarningDelayMerkleUpdate'):
102 self.WarningDelayMerkleUpdate = self.WarningDelay
104 self.lastMerkleUpdate = 0
105 self.nextMerkleUpdate = 0
108 self.updateMerkleTree()
110 def updateBlock(self, newBlock, height = None, bits = None, _HBH = None):
111 if newBlock == self.currentBlock[0]:
112 if height in (None, self.currentBlock[1]) and bits in (None, self.currentBlock[2]):
114 if not self.currentBlock[2] is None:
115 self.logger.error('Was working on block with wrong specs: %s (height: %d->%d; bits: %s->%s' % (
116 b2a_hex(newBlock[::-1]).decode('utf8'),
117 self.currentBlock[1],
119 b2a_hex(self.currentBlock[2][::-1]).decode('utf8'),
120 b2a_hex(bits[::-1]).decode('utf8'),
123 # Old block is invalid
124 self.currentMerkleTree = self.clearMerkleTree
125 if self.currentBlock[0] != newBlock:
126 self.lastBlock = self.currentBlock
129 height = self.currentBlock[1] + 1
131 if height % self.DifficultyChangeMod == 1 or self.currentBlock[2] is None:
132 self.logger.warning('New block: %s (height %d; bits: UNKNOWN)' % (b2a_hex(newBlock[::-1]).decode('utf8'), height))
133 # Pretend to be 1 lower height, so we possibly retain nextMerkleRoots
134 self.currentBlock = (None, height - 1, None)
135 self.clearMerkleRoots = Queue(0)
136 self.merkleRoots.clear()
139 bits = self.currentBlock[2]
142 _HBH = (b2a_hex(newBlock[::-1]).decode('utf8'), b2a_hex(bits[::-1]).decode('utf8'))
143 self.logger.info('New block: %s (height: %d; bits: %s)' % (_HBH[0], height, _HBH[1]))
144 self.currentBlock = (newBlock, height, bits)
146 if self.currentBlock[1] != height:
147 if self.currentBlock[1] == height - 1:
148 self.clearMerkleRoots = self.nextMerkleRoots
149 self.logger.debug('Adopting next-height clear merkleroots :)')
151 if self.currentBlock[1]:
152 self.logger.warning('Change from height %d->%d; no longpoll merkleroots available!' % (self.currentBlock[1], height))
153 self.clearMerkleRoots = Queue(self.WorkQueueSizeClear[1])
154 self.nextMerkleRoots = Queue(self._MaxClearSize)
156 self.logger.debug('Already using clear merkleroots for this height')
157 self.merkleRoots.clear()
162 def _trimBlock(self, MP, txnlist, txninfo, floodn, msgf):
163 fee = txninfo[-1].get('fee', None)
165 raise self._floodCritical(now, floodn, doin=msgf('fees unknown'))
167 # FIXME: coinbasevalue is *not* guaranteed to exist here
168 MP['coinbasevalue'] -= fee
175 # Aggressive "Power Of Two": Remove transactions even with fees to reach our goal
176 def _APOT(self, txninfopot, MP, POTInfo):
179 for txn in txninfopot:
180 if txn.get('fee') is None:
181 self._floodWarning(now, 'APOT-No-Fees', doin='Upstream didn\'t provide fee information required for aggressive POT', logf=self.logger.info)
185 feesTrimmed += txn['fee']
187 MP['coinbasevalue'] -= feesTrimmed
189 POTInfo[2] = [feeTxnsTrimmed, feesTrimmed]
190 self._floodWarning(now, 'POT-Trimming-Fees', doin='Aggressive POT trimming %d transactions with %d.%08d BTC total fees' % (feeTxnsTrimmed, feesTrimmed//100000000, feesTrimmed % 100000000), logf=self.logger.debug)
194 def _makeBlockSafe(self, MP, txnlist, txninfo):
195 blocksize = sum(map(len, txnlist)) + 80
196 while blocksize > 934464: # 1 "MB" limit - 64 KB breathing room
197 txnsize = len(txnlist[-1])
198 self._trimBlock(MP, txnlist, txninfo, 'SizeLimit', lambda x: 'Making blocks over 1 MB size limit (%d bytes; %s)' % (blocksize, x))
201 # NOTE: This check doesn't work at all without BIP22 transaction obj format
202 blocksigops = sum(a.get('sigops', 0) for a in txninfo)
203 while blocksigops > 19488: # 20k limit - 0x200 breathing room
204 txnsigops = txninfo[-1]['sigops']
205 self._trimBlock(MP, txnlist, txninfo, 'SigOpLimit', lambda x: 'Making blocks over 20k SigOp limit (%d; %s)' % (blocksigops, x))
206 blocksigops -= txnsigops
208 # Aim to produce blocks with "Power Of Two" transaction counts
209 # This helps avoid any chance of someone abusing CVE-2012-2459 with them
210 POTMode = getattr(self, 'POT', 1)
211 txncount = len(txnlist) + 1
213 feetxncount = txncount
214 for i in range(txncount - 2, -1, -1):
215 if 'fee' not in txninfo[i] or txninfo[i]['fee']:
219 if getattr(self, 'Greedy', None):
220 # Aim to cut off extra zero-fee transactions on the end
221 # NOTE: not cutting out ones intermixed, in case of dependencies
222 idealtxncount = feetxncount
224 idealtxncount = txncount
226 pot = 2**int(log(idealtxncount, 2))
227 POTInfo = MP['POTInfo'] = [[idealtxncount, feetxncount, txncount], [pot, None], None]
228 if pot < idealtxncount:
229 if pot * 2 <= txncount:
231 elif pot >= feetxncount:
233 elif POTMode > 1 and self._APOT(txninfo[pot-1:], MP, POTInfo):
234 # Trimmed even transactions with fees
238 self._floodWarning(now, 'Non-POT', doin='Making merkle tree with %d transactions (ideal: %d; max: %d)' % (pot, idealtxncount, txncount))
244 def updateMerkleTree(self):
246 self.logger.debug('Polling bitcoind for memorypool')
247 self.nextMerkleUpdate = now + self.TxnUpdateRetryWait
250 # First, try BIP 22 standard getblocktemplate :)
251 MP = self.access.getblocktemplate(self.GBTReq)
255 # Failing that, give BIP 22 draft (2012-02 through 2012-07) getmemorypool a chance
256 MP = self.access.getmemorypool(self.GMPReq)
259 # Finally, fall back to bitcoind 0.5/0.6 getmemorypool
260 MP = self.access.getmemorypool()
264 # This way, we get the error from the BIP22 call if the old one fails too
267 # Pre-BIP22 server (bitcoind <0.7 or Eloipool <20120513)
270 self.logger.warning('Upstream server is not BIP 22 compatible')
274 prevBlock = bytes.fromhex(MP['previousblockhash'])[::-1]
276 height = MP['height']
278 height = self.access.getinfo()['blocks'] + 1
279 bits = bytes.fromhex(MP['bits'])[::-1]
280 if (prevBlock, height, bits) != self.currentBlock:
281 self.updateBlock(prevBlock, height, bits, _HBH=(MP['previousblockhash'], MP['bits']))
283 txnlist = MP['transactions']
284 if len(txnlist) and isinstance(txnlist[0], dict):
286 txnlist = tuple(a['data'] for a in txnlist)
289 elif 'transactionfees' in MP:
290 # Backward compatibility with pre-BIP22 gmp_fees branch
291 txninfo = [{'fee':a} for a in MP['transactionfees']]
293 # Backward compatibility with pre-BIP22 hex-only (bitcoind <0.7, Eloipool <future)
294 txninfo = [{}] * len(txnlist)
295 # TODO: cache Txn or at least txid from previous merkle roots?
296 txnlist = [a for a in map(bytes.fromhex, txnlist)]
298 self._makeBlockSafe(MP, txnlist, txninfo)
300 cbtxn = self.makeCoinbaseTxn(MP['coinbasevalue'])
301 cbtxn.setCoinbase(b'\0\0')
303 txnlist.insert(0, cbtxn.data)
305 txnlist = [a for a in map(Txn, txnlist[1:])]
306 txnlist.insert(0, cbtxn)
307 txnlist = list(txnlist)
308 newMerkleTree = MerkleTree(txnlist)
309 if newMerkleTree.merkleRoot() != self.currentMerkleTree.merkleRoot():
310 newMerkleTree.POTInfo = MP.get('POTInfo')
311 newMerkleTree.oMP = oMP
313 if (not self.OldGMP) and 'proposal' in MP.get('capabilities', ()):
314 (prevBlock, height, bits) = self.currentBlock
315 coinbase = self.makeCoinbase(height=height)
316 cbtxn.setCoinbase(coinbase)
318 merkleRoot = newMerkleTree.merkleRoot()
319 MRD = (merkleRoot, newMerkleTree, coinbase, prevBlock, bits)
320 blkhdr = MakeBlockHeader(MRD)
321 data = assembleBlock(blkhdr, txnlist)
322 propose = self.access.getblocktemplate({
324 "data": b2a_hex(data).decode('utf8'),
327 self.logger.debug('Updating merkle tree (upstream accepted proposal)')
328 self.currentMerkleTree = newMerkleTree
330 self.RejectedProposal = (newMerkleTree, propose)
332 propose = propose['reject-reason']
335 self.logger.error('Upstream rejected proposed block: %s' % (propose,))
337 self.logger.debug('Updating merkle tree (no proposal support)')
338 self.currentMerkleTree = newMerkleTree
340 self.lastMerkleUpdate = now
341 self.nextMerkleUpdate = now + self.MinimumTxnUpdateWait
343 if self.needMerkle == 2:
345 self.needMerkleSince = now
347 def makeCoinbase(self, height):
349 if now > _makeCoinbase[0]:
350 _makeCoinbase[0] = now
353 _makeCoinbase[1] += 1
354 rv = self.CoinbasePrefix
355 rv += pack('>L', now) + pack('>Q', _makeCoinbase[1]).lstrip(b'\0')
356 # NOTE: Not using varlenEncode, since this is always guaranteed to be < 100
357 rv = bytes( (len(rv),) ) + rv
358 for v in self.CoinbaseAux.values():
362 if self.overflowed < t - 300:
363 self.logger.warning('Overflowing coinbase data! %d bytes long' % (len(rv),))
365 self.isOverflowed = True
368 self.isOverflowed = False
369 rv = bitcoin.script.encodeUNum(height) + rv
372 def makeMerkleRoot(self, merkleTree, height):
373 cbtxn = merkleTree.data[0]
374 cb = self.makeCoinbase(height=height)
375 cbtxn.setCoinbase(cb)
377 merkleRoot = merkleTree.merkleRoot()
378 return (merkleRoot, merkleTree, cb)
381 def _doing(self, what):
382 if self._doing_last == what:
387 self.logger.debug("Switching from (%4dx in %5.3f seconds) %s => %s" % (self._doing_i, now - self._doing_s, self._doing_last, what))
388 self._doing_last = what
392 def _floodWarning(self, now, wid, wmsgf = None, doin = True, logf = None):
394 doin = self._doing_last
396 return lambda: "%s (doing %s)" % (f(), doin)
398 winfo = self.lastWarning.setdefault(wid, [0, None])
399 (lastTime, lastDoing) = winfo
400 if now <= lastTime + max(5, self.MinimumTxnUpdateWait):
406 logf = self.logger.warning
407 logf(wmsgf() if wmsgf else doin)
409 def _makeOne(self, putf, merkleTree, height):
410 MT = self.currentMerkleTree
411 height = self.currentBlock[1]
412 MR = self.makeMerkleRoot(MT, height=height)
413 # Only add it if the height hasn't changed in the meantime, to avoid a race
414 if self.currentBlock[1] == height:
418 self._doing('clear merkle roots')
419 self._makeOne(self.clearMerkleRoots.put, self.clearMerkleTree, height=self.currentBlock[1])
422 self._doing('longpoll merkle roots')
423 self._makeOne(self.nextMerkleRoots.put, self.clearMerkleTree, height=self.currentBlock[1] + 1)
425 def makeRegular(self):
426 self._doing('regular merkle roots')
427 self._makeOne(self.merkleRoots.append, self.currentMerkleTree, height=self.currentBlock[1])
429 def merkleMaker_II(self):
432 # No bits = no mining :(
433 if self.currentBlock[2] is None:
434 return self.updateMerkleTree()
436 # First, ensure we have the minimum clear, next, and regular (in that order)
437 if self.clearMerkleRoots.qsize() < self.WorkQueueSizeClear[0]:
438 return self.makeClear()
439 if self.nextMerkleRoots.qsize() < self.WorkQueueSizeLongpoll[0]:
440 return self.makeNext()
441 if len(self.merkleRoots) < self.WorkQueueSizeRegular[0]:
442 return self.makeRegular()
444 # If we've met the minimum requirements, consider updating the merkle tree
445 if self.nextMerkleUpdate <= now:
446 return self.updateMerkleTree()
448 # Finally, fill up clear, next, and regular until we've met the maximums
449 if self.clearMerkleRoots.qsize() < self.WorkQueueSizeClear[1]:
450 return self.makeClear()
451 if self.nextMerkleRoots.qsize() < self.WorkQueueSizeLongpoll[1]:
452 return self.makeNext()
453 if len(self.merkleRoots) < self.WorkQueueSizeRegular[1] or self.merkleRoots[0][1] != self.currentMerkleTree:
454 return self.makeRegular()
456 # Nothing left to do, fire onBlockUpdate event (if appropriate) and sleep
457 if self.needMerkle == 1:
459 self.needMerkle = False
461 # TODO: rather than sleepspin, block until MinimumTxnUpdateWait expires or threading.Condition(?)
462 sleep(self.IdleSleepTime)
464 def merkleMaker_I(self):
468 self.merkleMaker_II()
470 if self.needMerkle == 1 and now > self.needMerkleSince + self.WarningDelayTxnLongpoll:
471 self._floodWarning(now, 'NeedMerkle', lambda: 'Transaction-longpoll requested %d seconds ago, and still not ready. Is your server fast enough to keep up with your configured WorkQueueSizeRegular maximum?' % (now - self.needMerkleSince,))
472 if now > self.nextMerkleUpdate + self.WarningDelayMerkleUpdate:
473 self._floodWarning(now, 'MerkleUpdate', lambda: "Haven't updated the merkle tree in at least %d seconds! Is your server fast enough to keep up with your configured work queue minimums?" % (now - self.lastMerkleUpdate,))
480 self.logger.critical(traceback.format_exc())
482 def start(self, *a, **k):
484 super().start(*a, **k)
488 MRD = self.merkleRoots.pop()
489 self.LowestMerkleRoots = min(len(self.merkleRoots), self.LowestMerkleRoots)
492 qsz = self.clearMerkleRoots.qsize()
494 self.logger.warning('clearMerkleRoots running out! only %d left' % (qsz,))
495 MRD = self.clearMerkleRoots.get()
496 self.LowestClearMerkleRoots = min(self.clearMerkleRoots.qsize(), self.LowestClearMerkleRoots)
498 (merkleRoot, merkleTree, cb) = MRD
499 (prevBlock, height, bits) = self.currentBlock
500 return (merkleRoot, merkleTree, cb, prevBlock, bits, rollPrevBlk)
502 def getMC(self, wantClear = False):
503 (prevBlock, height, bits) = self.currentBlock
504 mt = self.clearMerkleTree if wantClear else self.currentMerkleTree
505 cb = self.makeCoinbase(height=height)
506 rollPrevBlk = (mt == self.clearMerkleTree)
507 return (height, mt, cb, prevBlock, bits, rollPrevBlk)
514 reallogger = MM.logger
517 def critical(self, *a):
518 if self.LO > 1: return
519 reallogger.critical(*a)
520 def warning(self, *a):
522 reallogger.warning(*a)
525 MM.logger = fakelogger()
529 # _makeBlockSafe tests
530 from copy import deepcopy
534 txnlist = [b'\0', b'\x01', b'\x02']
535 txninfo = [{'fee':0, 'sigops':1}, {'fee':5, 'sigops':10000}, {'fee':0, 'sigops':10001}]
537 m = deepcopy( (MP, txnlist, txninfo) )
540 MM._makeBlockSafe(*m)
545 assert LO < 2 # An expected error wasn't thrown
546 if 'POTInfo' in m[0]:
550 assert MBS() == (MP, txnlist[:2], txninfo[:2])
551 txninfo[2]['fee'] = 1
553 MPx['coinbasevalue'] -= 1
554 assert MBS() == (MPx, txnlist[:2], txninfo[:2])
555 txninfo[2]['sigops'] = 1
556 assert MBS(1) == (MP, txnlist, txninfo)
559 txnlist.append(b'\x03')
560 txninfo.append({'fee':1, 'sigops':0})
562 MPx['coinbasevalue'] -= 1
563 assert MBS() == (MPx, txnlist[:3], txninfo[:3])