Merge branch 'ambig_t'
[bitcoin:eloipool.git] / merklemaker.py
1 # Eloipool - Python Bitcoin pool server
2 # Copyright (C) 2011-2012  Luke Dashjr <luke-jr+eloipool@utopios.org>
3 #
4 # This program is free software: you can redistribute it and/or modify
5 # it under the terms of the GNU Affero General Public License as
6 # published by the Free Software Foundation, either version 3 of the
7 # License, or (at your option) any later version.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12 # GNU Affero General Public License for more details.
13 #
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
16
17 from binascii import a2b_hex, b2a_hex
18 from bitcoin.script import countSigOps
19 from bitcoin.txn import Txn
20 from collections import deque
21 from queue import Queue
22 import jsonrpc
23 import logging
24 from merkletree import MerkleTree
25 from struct import pack
26 import threading
27 from time import sleep, time
28 import traceback
29
30 _makeCoinbase = [0, 0]
31
32 class merkleMaker(threading.Thread):
33         def __init__(self, *a, **k):
34                 super().__init__(*a, **k)
35                 self.daemon = True
36                 self.logger = logging.getLogger('merkleMaker')
37                 self.CoinbasePrefix = b''
38                 self.CoinbaseAux = {}
39                 self.isOverflowed = False
40                 self.overflowed = 0
41         
42         def _prepare(self):
43                 self.access = jsonrpc.ServiceProxy(self.UpstreamURI)
44                 
45                 self.currentBlock = (None, None)
46                 self.currentMerkleTree = None
47                 self.merkleRoots = deque(maxlen=self.WorkQueueSizeRegular[1])
48                 self.LowestMerkleRoots = self.WorkQueueSizeRegular[1]
49                 self.clearMerkleTree = MerkleTree([self.clearCoinbaseTxn])
50                 self.clearMerkleRoots = Queue(self.WorkQueueSizeLongpoll[1])
51                 self.LowestClearMerkleRoots = self.WorkQueueSizeLongpoll[1]
52                 
53                 self.nextMerkleUpdate = 0
54                 global now
55                 now = time()
56                 self.updateMerkleTree()
57         
58         def updateBlock(self, newBlock, bits = None, _HBH = None):
59                 if newBlock == self.currentBlock[0]:
60                         if bits in (None, self.currentBlock[1]):
61                                 return
62                         self.logger.error('Was working on block with wrong specs: %s (bits: %s->%s)' % (
63                                 b2a_hex(newBlock[::-1]).decode('utf8'),
64                                 b2a_hex(self.currentBlock[1][::-1]).decode('utf8'),
65                                 b2a_hex(bits[::-1]).decode('utf8'),
66                         ))
67                 
68                 if bits is None:
69                         bits = self.currentBlock[1]
70                 if _HBH is None:
71                         _HBH = (b2a_hex(newBlock[::-1]).decode('utf8'), b2a_hex(bits[::-1]).decode('utf8'))
72                 self.logger.info('New block: %s (bits: %s)' % _HBH)
73                 self.merkleRoots.clear()
74                 self.currentMerkleTree = self.clearMerkleTree
75                 if self.currentBlock[0] != newBlock:
76                         self.lastBlock = self.currentBlock
77                 self.currentBlock = (newBlock, bits)
78                 self.needMerkle = 2
79                 self.onBlockChange()
80         
81         def updateMerkleTree(self):
82                 global now
83                 self.logger.debug('Polling bitcoind for memorypool')
84                 self.nextMerkleUpdate = now + self.TxnUpdateRetryWait
85                 MP = self.access.getmemorypool()
86                 prevBlock = a2b_hex(MP['previousblockhash'])[::-1]
87                 bits = a2b_hex(MP['bits'])[::-1]
88                 if (prevBlock, bits) != self.currentBlock:
89                         self.updateBlock(prevBlock, bits, _HBH=(MP['previousblockhash'], MP['bits']))
90                 # TODO: cache Txn or at least txid from previous merkle roots?
91                 txnlist = [a for a in map(a2b_hex, MP['transactions'])]
92                 
93                 cbtxn = self.makeCoinbaseTxn(MP['coinbasevalue'])
94                 cbtxn.setCoinbase(b'\0\0')
95                 cbtxn.assemble()
96                 txnlist.insert(0, cbtxn.data)
97                 
98                 txnlistsz = sum(map(len, txnlist))
99                 while txnlistsz > 934464:  # TODO: 1 "MB" limit - 64 KB breathing room
100                         self.logger.debug('Trimming transaction for size limit')
101                         txnlistsz -= len(txnlist.pop())
102                 
103                 txnlistsz = sum(map(countSigOps, txnlist))
104                 while txnlistsz > 19488:  # TODO: 20k limit - 0x200 breathing room
105                         self.logger.debug('Trimming transaction for SigOp limit')
106                         txnlistsz -= countSigOps(txnlist.pop())
107                 
108                 txnlist = [a for a in map(Txn, txnlist[1:])]
109                 txnlist.insert(0, cbtxn)
110                 txnlist = list(txnlist)
111                 newMerkleTree = MerkleTree(txnlist)
112                 if newMerkleTree.merkleRoot() != self.currentMerkleTree.merkleRoot():
113                         self.logger.debug('Updating merkle tree')
114                         self.currentMerkleTree = newMerkleTree
115                 self.nextMerkleUpdate = now + self.MinimumTxnUpdateWait
116                 
117                 if self.needMerkle == 2:
118                         self.needMerkle = 1
119         
120         def makeCoinbase(self):
121                 now = int(time())
122                 if now > _makeCoinbase[0]:
123                         _makeCoinbase[0] = now
124                         _makeCoinbase[1] = 0
125                 else:
126                         _makeCoinbase[1] += 1
127                 rv = self.CoinbasePrefix
128                 rv += pack('>L', now) + pack('>Q', _makeCoinbase[1]).lstrip(b'\0')
129                 for v in self.CoinbaseAux.values():
130                         rv += v
131                 if len(rv) > 100:
132                         t = time()
133                         if self.overflowed < t - 300:
134                                 self.logger.warning('Overflowing coinbase data! %d bytes long' % (len(rv),))
135                                 self.overflowed = t
136                                 self.isOverflowed = True
137                         rv = rv[:100]
138                 else:
139                         self.isOverflowed = False
140                 return rv
141         
142         def makeMerkleRoot(self, merkleTree):
143                 cbtxn = merkleTree.data[0]
144                 cb = self.makeCoinbase()
145                 cbtxn.setCoinbase(cb)
146                 cbtxn.assemble()
147                 merkleRoot = merkleTree.merkleRoot()
148                 return (merkleRoot, merkleTree, cb)
149         
150         _doing_last = None
151         def _doing(self, what):
152                 if self._doing_last == what:
153                         self._doing_i += 1
154                         return
155                 global now
156                 if self._doing_last:
157                         self.logger.debug("Switching from (%4dx in %5.3f seconds) %s => %s" % (self._doing_i, now - self._doing_s, self._doing_last, what))
158                 self._doing_last = what
159                 self._doing_i = 1
160                 self._doing_s = now
161         
162         def merkleMaker_I(self):
163                 global now
164                 
165                 # First, update merkle tree if we haven't for a while and aren't crunched for time
166                 now = time()
167                 if self.nextMerkleUpdate <= now and self.clearMerkleRoots.qsize() > self.WorkQueueSizeLongpoll[0] and len(self.merkleRoots) > self.WorkQueueSizeRegular[0]:
168                         self.updateMerkleTree()
169                 # Next, fill up the longpoll queue first, since it can be used as a failover for the main queue
170                 elif not self.clearMerkleRoots.full():
171                         self._doing('blank merkle roots')
172                         self.clearMerkleRoots.put(self.makeMerkleRoot(self.clearMerkleTree))
173                 # Next, fill up the main queue (until they're all current)
174                 elif len(self.merkleRoots) < self.WorkQueueSizeRegular[1] or self.merkleRoots[0][1] != self.currentMerkleTree:
175                         self._doing('regular merkle roots')
176                         self.merkleRoots.append(self.makeMerkleRoot(self.currentMerkleTree))
177                 else:
178                         if self.needMerkle == 1:
179                                 self.onBlockUpdate()
180                                 self.needMerkle = False
181                         self._doing('idle')
182                         # TODO: rather than sleepspin, block until MinimumTxnUpdateWait expires or threading.Condition(?)
183                         sleep(self.IdleSleepTime)
184         
185         def run(self):
186                 while True:
187                         try:
188                                 self.merkleMaker_I()
189                         except:
190                                 self.logger.critical(traceback.format_exc())
191         
192         def start(self, *a, **k):
193                 self._prepare()
194                 super().start(*a, **k)
195         
196         def getMRD(self):
197                 (prevBlock, bits) = self.currentBlock
198                 try:
199                         MRD = self.merkleRoots.pop()
200                         self.LowestMerkleRoots = min(len(self.merkleRoots), self.LowestMerkleRoots)
201                         rollPrevBlk = False
202                 except IndexError:
203                         qsz = self.clearMerkleRoots.qsize()
204                         if qsz < 0x10:
205                                 self.logger.warning('clearMerkleRoots running out! only %d left' % (qsz,))
206                         MRD = self.clearMerkleRoots.get()
207                         self.LowestClearMerkleRoots = min(self.clearMerkleRoots.qsize(), self.LowestClearMerkleRoots)
208                         rollPrevBlk = True
209                 (merkleRoot, merkleTree, cb) = MRD
210                 return (merkleRoot, merkleTree, cb, prevBlock, bits, rollPrevBlk)