Update serialization

catapult-docs-cli/README.md

@@ -62,29 +62,26 @@ Use ``--help`` to obtain help about the available options.
 
 1. Checkout the following git repositories:
 
-   - [catapult-docs-cli](git@github.com:symbol/catapult-docs-cli.git)
-   - [catbuffer-schemas](git@github.com:symbol/catbuffer-schemas.git)
-   - [catbuffer-parser](git@github.com:symbol/catbuffer-parser.git)
-   - [catapult-client](git@github.com:symbol/catapult-client.git)
    - [symbol-docs](git@github.com:symbol/symbol-docs.git)
+   - [symbol](git@github.com:symbol/symbol.git)
 
-2. From within ``catbuffer-parser`` run:
+2. From within ``symbol/catbuffer/parser`` run:
 
    ``` bash
-   python3 -m catparser -s ../catbuffer-schemas/symbol/all.cats -i ../catbuffer-schemas/symbol/ > ../catapult-docs-cli/symbol.yaml
+   python3 -m catparser -s ../schemas/symbol/all.cats -i ../schemas/symbol/ -o ../../../symbol-docs/catapult-docs-cli/symbol.yaml
    ```
 
    This will produce a ``symbol.yaml`` in the ``catapult-docs-cli`` folder.
 
-3. Go to the ``catapult-docs-cli`` folder and run:
+3. Go to the ``symbol-docs/catapult-docs-cli`` folder and run:
 
    ``` bash
-   python3 -m catapult_docs_cli.cli serialization
+   python3 -m catapult_docs_cli.cli serialization -d ../source/serialization/ -a ../../symbol/client/catapult/ -h ../../symbol/catbuffer/schemas/symbol/
    ```
 
    The default values are already configured to work with the above folder structure so you don't need to provide any additional parameters.
 
-   The default output folder is ``../symbol-docs/source/serialization``, and a lot of ``.html`` and one ``.rst`` file will be generated there.
+   The output folder is ``../source/serialization``, and a lot of ``.html`` and one ``.rst`` file will be generated there.
 
 ### Generate serialization docs for NEM
 

catapult-docs-cli/catapult_docs_cli/commands/serialization.py

@@ -1,6 +1,6 @@
 import re
 import os
-import ruamel.yaml as yaml
+from ruamel.yaml import YAML
 from markdown import markdown, Markdown
 from .base import Command
 from io import StringIO
@@ -142,6 +142,9 @@ def calc_total_type_size(self, element):
         else:
             # Structs
             for f in element['layout']:
+                if 'condition' in f:
+                    var = 1
+                    continue
                 if f['type'] == 'byte':
                     field_size = 1
                     field_var = 0
@@ -194,10 +197,14 @@ def parse_comment(self, comment):
                 # Separate any non-keyword chars (like parenthesis or punctuation) before looking words up
                 # (And special-case an optional ending 's' for things like "MosaicId's")
                 m = re.search(r'^([^a-zA-Z]*)([a-zA-Z]+)([^a-rt-zA-Z]*)$', word)
-                if not ignore_keywords and m and m.group(2) in self.types:
-                    output += m.group(1) + self.type_description(self.types[m.group(2)]) + m.group(3)
+                # Treat unversioned type names as synonyms for V1, so the old docs that use them continue working.
+                if not ignore_keywords and m and (m.group(2) in self.types or m.group(2) + 'V1' in self.types):
+                    name = m.group(2) if m.group(2) in self.types else m.group(2) + 'V1'
+                    output += m.group(1) + self.type_description(self.types[name]) + m.group(3)
                 elif word == '\\note':
                     output += '<br/><b>Note:</b>'
+                elif word == 'deprecated).':
+                    output += '<b>deprecated</b>).'
                 else:
                     output += word
                 output += ' '
@@ -212,6 +219,9 @@ def print_rst_header(self, element, index_file):
         """
         name = element['name']
         print('.. _{}:'.format(make_anchor(name)), file=index_file)
+        if name.endswith('V1'):
+            # Add unversioned labels too, for docs not referencing the new versioned names
+            print('.. _{}:'.format(make_anchor(name[:-2])), file=index_file)
         print(file=index_file)
         print(name, file=index_file)
         print('=' * len(name), file=index_file)
@@ -236,11 +246,11 @@ def print_html_header(self, element, size, var, html_file):
         print('    <tr><td class="side-info-icon">&varr;</td><td>Size: {}</td></tr>'.format(make_size_label(size, var)), file=html_file)
         if name in self.type_schema_locations:
             print('    <tr><td class="side-info-icon"><i class="fab fa-github"></i></td>'
-                '<td><a href="https://github.com/symbol/catbuffer-schemas/blob/main/{}/{}#L{}">schema</a></td></tr>'.format( \
+                '<td><a href="https://github.com/symbol/symbol/blob/main/catbuffer/schemas/symbol{}/{}#L{}">schema</a></td></tr>'.format( \
                     self.source_api, self.type_schema_locations[name][0], self.type_schema_locations[name][1]), file=html_file)
         if name in self.type_catapult_locations:
             print('    <tr><td class="side-info-icon"><i class="fab fa-github"></i></td>'
-                '<td><a href="https://github.com/symbol/catapult-client/blob/main/{}#L{}">catapult model</a></td></tr>'.format(
+                '<td><a href="https://github.com/symbol/symbol/blob/main/client/catapult/{}#L{}">catapult model</a></td></tr>'.format(
                     self.type_catapult_locations[name][0], self.type_catapult_locations[name][1]), file=html_file)
         print('    </table></div>', file=html_file)
         print(self.parse_comment(element['comments']), file=html_file)
@@ -304,6 +314,9 @@ def print_struct_content(self, element, indent, html_file):
             elif disposition == 'reserved':
                 comment = '<b>reserved</b> {}<br/>'.format(make_keyword(v['value']))
             comment += self.parse_comment(v['comments'])
+            if 'condition' in v:
+                comment += '<b>This field is only present if:</b><br/>{}'.format(
+                    make_keyword(v['condition'] + ' ' + v['condition_operation'] + ' ' + v['condition_value']))
             print('   <div{}>&nbsp;</div>'.format('' if indent < 1 else ' class="indentation-cell"'), file=html_file)
             print('   <div{}>&nbsp;</div>'.format('' if indent < 2 else ' class="indentation-cell"'), file=html_file)
             print('   <div{}>&nbsp;</div>'.format('' if indent < 3 else ' class="indentation-cell"'), file=html_file)
@@ -341,7 +354,7 @@ def output_rst(self):
         print('Serialization', file=index_file)
         print('#############', file=index_file)
         print(file=index_file)
-        print('The `catbuffer schemas <https://github.com/symbol/catbuffer-schemas>`_ repository defines how the different Symbol entities type should be serialized (for example, Transactions). In combination with the `catbuffer-generators <https://github.com/symbol/catbuffer-generators>`_ project, developers can generate builder classes for a given set of programming languages.', file=index_file)
+        print('The `catbuffer schemas <https://github.com/symbol/symbol/tree/main/catbuffer/schemas>`_ repository defines how the different Symbol entities type should be serialized (for example, Transactions). In combination with the catbuffer-generators project, developers can generate builder classes for a given set of programming languages.', file=index_file)
         print(file=index_file)
 
         # Hide level 4 headers from local TOC using CSS: there's too many of them and I could not find
@@ -484,8 +497,18 @@ def execute(self):
         # Read a single YAML file containing all schemas
         with open(self.config['schema']) as f:
             try:
-                self.schema = yaml.safe_load(f)
-            except yaml.YAMLError as exc:
+                yaml = YAML(typ='safe', pure=True)
+                self.schema = yaml.load(f)
+                # Make sure every element has a 'comments' field
+                for e in self.schema:
+                    if 'comments' not in e:
+                        e['comments'] = ''
+                    if 'layout' in e:
+                        for f in e['layout']:
+                            if 'comments' not in f:
+                                f['comments'] = ''
+
+            except Exception as exc:
                 print(exc)
                 return
         # Build types dictionary indexed by type name, for simpler access

source/concepts/block.rst

@@ -20,12 +20,6 @@ Nodes stores blocks in a serialized form as follows:
 
 **EntityType**: 0x8143
 
-**Inlines:**
-
-* :ref:`SizePrefixedEntity <sizeprefixedentity>`
-* :ref:`VerifiableEntity <verifiableentity>`
-* :ref:`EntityBody <entitybody>`
-
 .. _block-header:
 
 .. csv-table::

source/concepts/cryptography.rst

@@ -12,7 +12,7 @@ Key pair
 
 Elliptic curve cryptography is based on **key pairs**: a private and a matching public key. In particular, |codename| uses the `Twisted Edwards curve <https://en.wikipedia.org/wiki/Twisted_Edwards_curve>`__ with the digital signature algorithm named `Ed25519 <https://ed25519.cr.yp.to>`__ and hashing algorithm **SHA-512**:
 
-* **Private key**: A random 256-bit (32 byte) integer used to sign :ref:`entities <verifiableentity>` known by the owner.
+* **Private key**: A random 256-bit (32 byte) integer used to sign entities known by the owner.
 
 * **Public key**: A 256-bit (32 bytes) integer derived from the private key. It serves as the public identifier of the key pair and can be disseminated widely. It is used to prove that an entity was signed with the paired private key.
 

source/concepts/receipt.rst

@@ -57,7 +57,7 @@ Recorded receipts
     **Core**
     -------------------------------------------------------------------
     -------------------------------------------------------------------
-    ``0x2143`` ``Harvest_Fee``              :ref:`balancechangereceipt`
+    ``0x2143`` ``Harvest_Fee``              :ref:`harvestfeereceipt`
     The recipient, account and amount of fees received for harvesting a block. It is recorded when a block is :doc:`harvested <harvesting>`.
     -------------------------------------------------------------------
     ``0x5143`` ``Inflation``                :ref:`inflationreceipt`
@@ -75,46 +75,46 @@ Recorded receipts
     **Mosaic**
     -------------------------------------------------------------------
     -------------------------------------------------------------------
-    ``0x414D`` ``Mosaic_Expired``           :ref:`mosaicexpiryreceipt`
+    ``0x414D`` ``Mosaic_Expired``           :ref:`mosaicexpiredreceipt`
     The identifier of the mosaic expiring in this block. It is recorded when a :doc:`mosaic <mosaic>` lifetime elapses.
     -------------------------------------------------------------------
-    ``0x124D`` ``Mosaic_Rental_Fee``        :ref:`balancetransferreceipt`
+    ``0x124D`` ``Mosaic_Rental_Fee``        :ref:`mosaicrentalfeereceipt`
     The sender and recipient of the mosaic id and amount representing the cost of registering the mosaic. It is recorded when a mosaic is registered.
     -------------------------------------------------------------------
     **Namespace**
     -------------------------------------------------------------------
     -------------------------------------------------------------------
-    ``0x414E`` ``Namespace_Expired``        :ref:`namespaceexpiryreceipt`
+    ``0x414E`` ``Namespace_Expired``        :ref:`namespaceexpiredreceipt`
     The identifier of the namespace expiring in this block. It is recorded when the :doc:`namespace <namespace>` lifetime elapses.
     -------------------------------------------------------------------
-    ``0x424E`` ``Namespace_Deleted``        :ref:`namespaceexpiryreceipt`
+    ``0x424E`` ``Namespace_Deleted``        :ref:`namespaceexpiredreceipt`
     The identifier of the namespace deleted in this block. It is recorded when the :doc:`namespace <namespace>` grace period elapses.
     -------------------------------------------------------------------
-    ``0x134E`` ``Namespace_Rental_Fee``     :ref:`balancetransferreceipt`
+    ``0x134E`` ``Namespace_Rental_Fee``     :ref:`namespacerentalfeereceipt`
     The sender and recipient of the mosaic id and amount representing the cost of extending the namespace. It is recorded when a namespace is registered or its duration is extended.
     -------------------------------------------------------------------
     **HashLock**
     -------------------------------------------------------------------
     -------------------------------------------------------------------
-    ``0x3148`` ``LockHash_Created``         :ref:`balancechangereceipt`
+    ``0x3148`` ``LockHash_Created``         :ref:`lockhashcreatedfeereceipt`
     The lockhash sender, mosaic id and amount locked. It is recorded when a valid :ref:`hashlocktransaction` is announced.
     -------------------------------------------------------------------
-    ``0x2248`` ``LockHash_Completed``       :ref:`balancechangereceipt`
+    ``0x2248`` ``LockHash_Completed``       :ref:`lockhashcompletedfeereceipt`
     The hashlock sender, mosaic id and amount locked that is returned. It is recorded when an AggregateBondedTransaction linked to the hash completes.
     -------------------------------------------------------------------
-    ``0x2348`` ``LockHash_Expired``         :ref:`balancechangereceipt`
+    ``0x2348`` ``LockHash_Expired``         :ref:`lockhashexpiredfeereceipt`
     The account receiving the locked mosaic, the mosaic id and the amount. It is recorded when a lock hash expires.
     -------------------------------------------------------------------
     **SecretLock**
     -------------------------------------------------------------------
     -------------------------------------------------------------------
-    ``0x3152`` ``LockSecret_Created``       :ref:`balancechangereceipt`
+    ``0x3152`` ``LockSecret_Created``       :ref:`locksecretcreatedfeereceipt`
     The secretlock sender, mosaic id and amount locked. It is recorded when a valid :ref:`secretlocktransaction` is announced.
     -------------------------------------------------------------------
-    ``0x2252`` ``LockSecret_Completed``     :ref:`balancechangereceipt`
+    ``0x2252`` ``LockSecret_Completed``     :ref:`locksecretcompletedfeereceipt`
     The secretlock recipient, mosaic id and amount locked. It is recorded when a secretlock is proved.
     -------------------------------------------------------------------
-    ``0x2352`` ``LockSecret_Expired``       :ref:`balancechangereceipt`
+    ``0x2352`` ``LockSecret_Expired``       :ref:`locksecretexpiredfeereceipt`
     The account receiving the locked mosaic, the mosaic id and the amount. It is recorded when a secretlock expires.
     -------------------------------------------------------------------
     ========== ============================ ===========================

source/conf.py

@@ -358,6 +358,11 @@ def setup(app):
     r'https://par.nsf.gov/*', # unsafe legacy renegotiation disabled
     r'https://github.com/*', #  Anchor not found
     r'https://www.exodus.com/support/*', # 403 Client Error
+    r'https://davecube-com.medium.com/*', # 403 Client Error, but valid
+    r'https://medium.com/*', # 403 Client Error, but valid
+    r'https://www.npmjs.com/*', # 403 Client Error, but valid
+    r'http://www.gnu.org/*', # 403 Client Error, but valid
+    r'https://docutils.sourceforge.io/*' # 403 Client Error, but valid
 ]
 linkcheck_anchors_ignore = [r'L\d+']