block_desc add ts_range and ref_num

Author: shuwen5

ectypes/block_desc.py

@@ -41,16 +41,53 @@ def _mtime(ts=None):
     return int(ts)
 
 
+def _ts_range(ts_range=None):
+    if ts_range is None:
+        return ts_range
+
+    if ts_range[0] is not None:
+        ts_range[0] = str(ts_range[0])
+
+    if ts_range[1] is not None:
+        ts_range[1] = str(ts_range[1])
+
+    return ts_range
+
+
 class BlockDesc(FixedKeysDict):
 
     keys_default = dict(
         block_id=_block_id,
         size=int,
         range=_range,
+        ts_range=_ts_range,
         is_del=_is_del,
         mtime=_mtime,
+        ref_num=int,
     )
 
     def mark_del(self):
+        if self['ref_num'] != 0:
+            raise ValueError("cannot mark del block with ref_num:{n} > 0".format(
+                n=self['ref_num']))
+
         self["is_del"] = 1
         self.mtime = int(time.time())
+
+    def is_mark_del(self):
+        return self['is_del'] == 1
+
+    def add_ref(self):
+        if self['is_del'] != 0:
+            raise ValueError("reference a block marked delete")
+
+        self['ref_num'] += 1
+
+    def rm_ref(self):
+        if self['ref_num'] < 1:
+            raise ValueError("ref_num:{n} < 1".format(n=self['ref_num']))
+
+        self['ref_num'] -= 1
+
+    def can_del(self):
+        return self['ref_num'] == 0

ectypes/block_group.py

@@ -100,35 +100,69 @@ def make_type_map(self):
 
     def mark_delete_block(self, block_index):
         block = self.get_block(block_index, raise_error=True)
-        block.mark_del()
 
-        return block
-
-    def delete_block(self, block_index):
-        block = self.get_block(block_index, raise_error=True)
-        del self['blocks'][str(block_index)]
+        block.rm_ref()
+        if block.can_del():
+            block.mark_del()
 
         return block
 
     def mark_delete_block_byid(self, block_id):
         block = self.get_block_byid(block_id, raise_error=True)
-        block.mark_del()
+
+        block.rm_ref()
+        if block.can_del():
+            block.mark_del()
 
         return block
 
-    def delete_block_byid(self, block_id):
+    def unlink_block(self, block_index):
+        block = self.get_block(block_index, raise_error=True)
+
+        if not block.is_mark_del():
+            block.rm_ref()
+
+        if block.can_del():
+            del self['blocks'][str(block_index)]
+
+        return block
+
+    def unlink_block_byid(self, block_id):
         block = self.get_block_byid(block_id, raise_error=True)
-        del self['blocks'][block_id.block_index]
+
+        if not block.is_mark_del():
+            block.rm_ref()
+
+        if block.can_del():
+            del self['blocks'][block_id.block_index]
 
         return block
 
+    def delete_block(self, block_index):
+        return self.unlink_block(block_index)
+
+    def delete_block_byid(self, block_id):
+        return self.unlink_block_byid(block_id)
+
     def has(self, block):
         bid = block['block_id']
         bidx = bid.block_index
 
         existent = self['blocks'].get(bidx)
         return existent == block
 
+    def link_block(self, block_index):
+        block = self.get_block(block_index, raise_error=True)
+
+        block.add_ref()
+        return block
+
+    def link_block_byid(self, block_id):
+        block = self.get_block_byid(block_id, raise_error=True)
+
+        block.add_ref()
+        return block
+
     def add_block(self, new_block, replace=False, allow_exist=False):
 
         if self.has(new_block) and allow_exist:
@@ -251,7 +285,7 @@ def classify_blocks(self, idc_index, only_primary=True):
             if blk is None:
                 continue
 
-            if blk['is_del'] == 1:
+            if blk.is_mark_del():
                 mark_del.append(blk)
                 continue
 

ectypes/test/test_block_desc.py

@@ -19,24 +19,34 @@ def test_blockdesc(self):
                   {'block_id': None,
                    'size': 0,
                    'range': None,
+                   'ts_range': None,
+                   'ref_num': 0,
                    'is_del': 0}),
 
                  ({'block_id': block_id,
                    'range': ['a', 'b'],
+                   'ts_range': ["124", None],
+                   'ref_num': 0,
                    'size': 34,
                    'mtime': 1,
                    'is_del': 0},
                   {'block_id': BlockID(block_id),
                    'range': rangeset.Range('a', 'b'),
+                   'ts_range': ["124", None],
+                   'ref_num': 0,
                    'size': 34,
                    'mtime': 1,
                    'is_del': 0}),
 
                  ({'block_id': BlockID(block_id),
                    'range': rangeset.Range('b', 'bb'),
+                   'ts_range': ["1235", "456"],
+                   'ref_num': 0,
                    'mtime': 1},
                   {'block_id': BlockID(block_id),
                    'range': rangeset.Range('b', 'bb'),
+                   'ts_range': ["1235", "456"],
+                   'ref_num': 0,
                    'size': 0,
                    'mtime': 1,
                    'is_del': 0, })
@@ -61,15 +71,48 @@ def test_json(self):
                                     DriveID('idc000' 'c62d8736c7280002'), 1),
             'size': 1000,
             'range': ['0a', '0b'],
+            'ts_range': ["1235", "456"],
+            'ref_num': 1,
             'is_del': 0,
             'mtime': 1,
 
         })
 
         rst = utfjson.dump(blk)
-        expected = ('{"is_del": 0, "range": ["0a", "0b"], "mtime": 1, "block_id": '
-                    '"d0g0006400000001230000idc000c62d8736c72800020000000001", "size": 1000}')
+
+        expected = ('{"block_id": "d0g0006400000001230000idc000c62d8736c72800020000000001", "is_del": 0, "ref_num": 1, "range": ["0a", "0b"], "mtime": 1, "ts_range": ["1235", "456"], "size": 1000}')
 
         self.assertEqual(expected, rst)
         loaded = BlockDesc(utfjson.load(rst))
         self.assertEqual(blk, loaded)
+
+    def test_ref(self):
+        block_id = 'd1g0006300000001230101idc000c62d8736c72800020000000001'
+        blk = BlockDesc({
+            'block_id': block_id,
+            'size': 1000,
+            'range': ['0a', '0b'],
+            'ts_range': ["1235", "456"],
+            'ref_num': 1,
+            'is_del': 0,
+            'mtime': 1,
+
+        })
+
+        blk.add_ref()
+        blk.add_ref()
+        self.assertEqual(blk['ref_num'], 3)
+
+        blk.rm_ref()
+        self.assertEqual(blk['ref_num'], 2)
+
+        self.assertRaises(ValueError, blk.mark_del)
+
+        blk.rm_ref()
+        blk.rm_ref()
+        self.assertEqual(blk['ref_num'], 0)
+        self.assertTrue(blk.can_del())
+
+        self.assertRaises(ValueError, blk.rm_ref)
+
+        blk.mark_del()

ectypes/test/test_block_group.py

@@ -77,6 +77,8 @@ def setUp(self):
                                     DriveID('idc000' 'c62d8736c7280002'), 1),
             'size': 1000,
             'range': ['0a', '0b'],
+            'ts_range': None,
+            'ref_num': 1,
             'is_del': 0
         })
 
@@ -150,6 +152,13 @@ def test_mark_delete_block(self):
         g = BlockGroup(block_group_id='g000640000000123', idcs=['a', 'b', 'c'], config=_ec_config)
 
         g.add_block(self.foo_block)
+
+        self.foo_block.add_ref()
+        del_blk = g.mark_delete_block('0000')
+        self.assertDictEqual(del_blk, g.get_block('0000'))
+        self.assertEqual(del_blk['ref_num'], 1)
+        self.assertEqual(0, del_blk['is_del'])
+
         del_blk = g.mark_delete_block('0000')
         self.assertDictEqual(del_blk, g.get_block('0000'))
 
@@ -161,6 +170,12 @@ def test_mark_delete_block_byid(self):
         g = BlockGroup(block_group_id='g000640000000123', idcs=['a', 'b', 'c'], config=_ec_config)
 
         g.add_block(self.foo_block)
+
+        self.foo_block.add_ref()
+        del_blk = g.mark_delete_block_byid(self.foo_block['block_id'])
+        self.assertDictEqual(del_blk, g.get_block_byid(self.foo_block['block_id']))
+        self.assertEqual(del_blk['ref_num'], 1)
+
         del_blk = g.mark_delete_block_byid(self.foo_block['block_id'])
         self.assertDictEqual(del_blk, g.get_block_byid(self.foo_block['block_id']))
 
@@ -181,6 +196,10 @@ def test_delete_block(self):
         g.add_block(self.foo_block)
         self.assertIsNotNone(g.get_block('0000'))
 
+        self.foo_block.add_ref()
+        del_blk = g.delete_block('0000')
+        self.assertIsNotNone(g.get_block('0000', raise_error=False))
+
         del_blk = g.delete_block('0000')
         self.assertIsNone(g.get_block('0000', raise_error=False))
         self.assertDictEqual(self.foo_block, del_blk)
@@ -397,6 +416,8 @@ def make_test_block_group(self, blk_idxes, config=None):
         base_blk = BlockDesc({
             'size': 1000,
             'range': ['0a', '0b'],
+            'ts_range': ["123", "456"],
+            'ref_num': 1,
             'is_del': 0
         })
 
@@ -541,7 +562,7 @@ def test_get_block_byid(self):
         self.assertRaises(BlockNotFoundError, bg.get_block_byid, fake_bid, True)
         self.assertRaises(BlockNotFoundError, bg.get_block_byid, fake_bid)
 
-    def test_delete_block_byid(self):
+    def test_unlink_block_byid(self):
 
         blk_idxes = ['0000', '0001', '0002', '0003', '0008', '0012']
 
@@ -550,6 +571,15 @@ def test_delete_block_byid(self):
         blks = bg.indexes_to_blocks(blk_idxes)
         bids = [blk['block_id'] for blk in blks]
 
+        bg.link_block_byid(bids[1])
+        self.assertEqual(blks[1]['ref_num'], 2)
+
+        del_blk = bg.delete_block_byid(bids[1])
+        self.assertDictEqual(del_blk, blks[1])
+
+        del_blk = bg.mark_delete_block_byid(bids[1])
+        self.assertEqual(del_blk["is_del"], 1)
+
         del_blk = bg.delete_block_byid(bids[1])
         self.assertDictEqual(del_blk, blks[1])
 
@@ -562,6 +592,27 @@ def test_delete_block_byid(self):
 
         self.assertListEqual(blks, act_blks)
 
+    def test_link_block(self):
+
+        blk_idxes = ['0000', '0001', '0002', '0003', '0008', '0012']
+
+        bg = self.make_test_block_group(blk_idxes)
+        blks = bg.indexes_to_blocks(blk_idxes)
+
+        args = [blks[1], False, False]
+        self.assertRaises(BlockExists, bg.add_block, *args)
+        act_blks = bg.indexes_to_blocks(blk_idxes)
+        self.assertListEqual(blks, act_blks)
+
+        bg.link_block(blk_idxes[1])
+        self.assertEqual(blks[1]['ref_num'], 2)
+
+        bg.unlink_block(blk_idxes[1])
+        self.assertDictEqual(blks[1], bg.add_block(blks[1], allow_exist=True))
+
+        act_blks = bg.indexes_to_blocks(blk_idxes)
+        self.assertListEqual(blks, act_blks)
+
     def test_add_block(self):
 
         blk_idxes = ['0000', '0001', '0002', '0003', '0008', '0012']