Dev Docs Components CNID

WIP - ALL LINKS IN THIS WIKI STRUCTURE ARE CURRENTLY BROKEN DURING WIKI MIGRATION

THESE ARE COMMUNITY DOCS

CNID Database System

Overview

The CNID (Catalog Node ID) system is a critical component of Netatalk that provides persistent file and directory identification. It solves the fundamental problem of maintaining stable file references across filesystem operations like renames, moves, and remounts, which is essential for Mac clients that expect files to maintain their identity.

Implementation Files

• include/atalk/cnid.h - CNID system interface definitions and structures
• libatalk/cnid/ - CNID backend implementations directory
• etc/cnid_dbd/ - CNID database daemon implementation
• etc/cnid_metad/ - CNID metadata coordinator daemon
• libatalk/cnid/cnid_init.c - CNID system initialization and module management

Pluggable Architecture

The CNID system is built on a sophisticated pluggable architecture that allows different backend implementations while providing a consistent interface to AFP operations.

Implementation Files

• libatalk/cnid/cnid_init.c - Module registration and backend discovery
• include/atalk/cnid.h - Plugin interface definitions
• libatalk/cnid/dbd/ - Berkeley DB backend implementation
• libatalk/cnid/mysql/ - MySQL backend implementation (if enabled)
• libatalk/cnid/sqlite/ - SQLite backend implementation (if enabled)

CNID Module System

/*
 * CNID module - represents particular CNID implementation
 */
struct _cnid_module {
    char *name;                          // Module name (e.g., "dbd", "mysql", "sqlite")
    struct list_head db_list;            // Bidirectional list for module management
    struct _cnid_db *(*cnid_open)(struct cnid_open_args *args);  // Open function
    uint32_t flags;                      // Module-specific capability flags
};

CNID Database Abstraction

The core abstraction layer uses function pointers for complete backend independence:

typedef struct _cnid_db {
    uint32_t      cnid_db_flags;         // Backend capability flags
    struct vol   *cnid_db_vol;           // Associated volume
    void         *cnid_db_private;       // Backend-specific data

    // Core operations (function pointers for pluggability)
    cnid_t (*cnid_add)(struct _cnid_db *cdb, const struct stat *st, cnid_t did,
                       const char *name, size_t, cnid_t hint);
    int (*cnid_delete)(struct _cnid_db *cdb, cnid_t id);
    cnid_t (*cnid_get)(struct _cnid_db *cdb, cnid_t did, const char *name, size_t);
    cnid_t (*cnid_lookup)(struct _cnid_db *cdb, const struct stat *st,
                          cnid_t did, const char *name, size_t);
    cnid_t (*cnid_nextid)(struct _cnid_db *cdb);
    char *(*cnid_resolve)(struct _cnid_db *cdb, cnid_t *id, void *buffer, size_t len);
    int (*cnid_update)(struct _cnid_db *cdb, cnid_t id, const struct stat *st,
                       cnid_t did, const char *name, size_t len);
    void (*cnid_close)(struct _cnid_db *cdb);
    int (*cnid_getstamp)(struct _cnid_db *cdb, void *buffer, const size_t len);
    cnid_t (*cnid_rebuild_add)(struct _cnid_db *, const struct stat *, cnid_t,
                               const char *, size_t, cnid_t);
    int (*cnid_find)(struct _cnid_db *cdb, const char *name, size_t namelen,
                     void *buffer, size_t buflen);
    int (*cnid_wipe)(struct _cnid_db *cdb);
} cnid_db;

CNID Backend Capability Flags

/* CNID object flags */
#define CNID_FLAG_PERSISTENT   0x01      // Implements DID persistence
#define CNID_FLAG_MANGLING     0x02      // Has name mangling feature
#define CNID_FLAG_SETUID       0x04      // Set db owner to parent folder owner
#define CNID_FLAG_BLOCK        0x08      // Block signals in update
#define CNID_FLAG_NODEV        0x10      // Don't use device number only inode
#define CNID_FLAG_LAZY_INIT    0x20      // Supports lazy initialization
#define CNID_FLAG_INODE        0x80      // In cnid_add the inode is authoritative

CNID Constants and Error Codes

#define CNID_INVALID   0        // Invalid CNID value
#define CNID_START     17       // First valid ID (IDs 1-16 reserved)

// Error codes
#define CNID_ERR_PARAM 0x80000001   // Parameter error
#define CNID_ERR_PATH  0x80000002   // Path error
#define CNID_ERR_DB    0x80000003   // Database error
#define CNID_ERR_CLOSE 0x80000004   // Database was not open
#define CNID_ERR_MAX   0x80000005   // Maximum error code

Backend Registration System

// Initialize the CNID backend system
void cnid_init(void);

// Register new CNID backend module
void cnid_register(struct _cnid_module *module);

// Open CNID database for volume
struct _cnid_db *cnid_open(struct vol *vol, char *type, int flags);

Architecture

Two-Tier Database Architecture

graph TB
    subgraph "AFP Clients"
        A[Mac Client 1]
        B[Mac Client 2]
        C[Mac Client N]
    end
    
    subgraph "AFP Layer"
        D[afpd Process 1]
        E[afpd Process 2]
        F[afpd Process N]
    end
    
    subgraph "CNID Coordination Layer"
        G[cnid_metad - Metadata Coordinator]
    end
    
    subgraph "CNID Database Layer"
        H[cnid_dbd - Database Daemon]
        I[Berkeley DB Files]
    end
    
    subgraph "Filesystem"
        J[Unix Filesystem]
        K[Volume Root]
    end
    
    A --> D
    B --> E
    C --> F
    D --> G
    E --> G
    F --> G
    G --> H
    H --> I
    H --> J
    I --> K

Loading

CNID System Operational Flow

The CNID system processes requests through a sophisticated 3-tier message passing architecture:

sequenceDiagram
    participant AFP as afpd Process
    participant Meta as cnid_metad
    participant DB as cnid_dbd
    participant BDB as Berkeley DB
    participant FS as Unix Filesystem
    
    Note over AFP,FS: CNID Lookup Operation
    
    AFP->>Meta: CNID Request (lookup by path)
    Meta->>DB: Route to appropriate volume daemon
    DB->>BDB: Database lookup by path hash
    BDB-->>DB: CNID + metadata
    DB-->>Meta: Operation result
    Meta-->>AFP: CNID response
    AFP->>FS: Use CNID for file operations
    
    Note over AFP,FS: CNID Add Operation
    
    AFP->>Meta: CNID Request (add new file)
    Meta->>DB: Route to volume daemon
    DB->>BDB: Generate new CNID
    DB->>BDB: Insert CNID record
    DB->>FS: Verify file still exists
    FS-->>DB: File confirmation
    BDB-->>DB: Insert success
    DB-->>Meta: New CNID assigned
    Meta-->>AFP: CNID creation result
    
    Note over AFP,FS: CNID Update Operation
    
    AFP->>Meta: CNID Request (update metadata)
    Meta->>DB: Route to volume daemon
    DB->>BDB: Update existing record
    DB->>FS: Verify file properties
    FS-->>DB: File status
    BDB-->>DB: Update success
    DB-->>Meta: Update result
    Meta-->>AFP: Update confirmation
    
    Note over Meta: Handles multiplexing & coordination
    Note over DB: Per-volume database daemon
    Note over BDB: Pluggable backend (dbd/mysql/sqlite)

Loading

CNID Backend Plugin Architecture

The pluggable backend system allows different database implementations while maintaining a consistent API:

graph TB
    subgraph "CNID API Layer"
        A[cnid_open]
        B[cnid_add]
        C[cnid_get]
        D[cnid_lookup]
        E[cnid_resolve]
        F[cnid_update]
        G[cnid_delete]
    end
    
    subgraph "Backend Registration"
        H[cnid_init - System Init]
        I[cnid_register - Module Registration]
        J[Module Discovery]
    end
    
    subgraph "Backend Implementations"
        K["dbd Backend<br/>Berkeley DB<br/>Local files"]
        L["mysql Backend<br/>MySQL Database<br/>Network storage"]
        M["sqlite Backend<br/>SQLite Database<br/>Embedded storage"]
    end
    
    subgraph "Backend Operations"
        N[Function Pointers<br/>cnid_add, cnid_get, etc.]
        O[Backend-specific Data<br/>cnid_db_private]
        P[Capability Flags<br/>PERSISTENT, MANGLING, etc.]
    end
    
    A --> N
    B --> N
    C --> N
    D --> N
    E --> N
    F --> N
    G --> N
    
    H --> I
    I --> J
    J --> K
    J --> L
    J --> M
    
    K --> N
    L --> N
    M --> N
    
    N --> O
    N --> P

Loading

CNID Error Handling and Recovery

stateDiagram-v2
    [*] --> Normal
    Normal --> DatabaseError: DB operation fails
    Normal --> PathError: Invalid path
    Normal --> ParameterError: Bad parameters
    
    DatabaseError --> Recovery: Retry operation
    Recovery --> Normal: Success
    Recovery --> Fatal: Max retries exceeded
    
    PathError --> Validation: Check filesystem
    Validation --> Normal: Path valid
    Validation --> Fatal: Path inaccessible
    
    ParameterError --> Normal: Parameter correction
    
    Fatal --> Disconnect: Close database
    Disconnect --> [*]: Cleanup complete
    
    Note right of DatabaseError: CNID_ERR_DB<br/>0x80000003
    Note right of PathError: CNID_ERR_PATH<br/>0x80000002
    Note right of ParameterError: CNID_ERR_PARAM<br/>0x80000001

Loading

Core Components

Implementation Files

• etc/cnid_metad/cnid_metad.c - Metadata coordinator daemon main implementation
• etc/cnid_dbd/cnid_dbd.c - Database daemon main implementation
• libatalk/cnid/dbd/cnid_dbd.c - Berkeley DB backend implementation
• libatalk/cnid/dbd/dbif.c - Database interface layer

1. cnid_metad - Metadata Coordinator

The metadata daemon serves as the coordination point for all CNID operations:

Implementation Files:

• etc/cnid_metad/cnid_metad.c - Main metadata daemon implementation
• etc/cnid_metad/cnid_metad.h - Metadata daemon structures and definitions
• libatalk/util/server_child.c - Child process management utilities

Responsibilities

• Process Management: Spawns and manages cnid_dbd processes
• Request Routing: Routes CNID requests to appropriate database daemons
• Connection Multiplexing: Handles multiple AFP daemon connections
• Resource Management: Manages database daemon lifecycle

Process Architecture

// Metadata daemon state
struct cnid_metad_state {
    int listen_fd;              // Unix domain socket for AFP connections
    struct pollfd *pollfds;     // Poll descriptors for connections
    int max_connections;        // Maximum concurrent connections
    
    // Database daemon management
    struct {
        pid_t pid;              // Database daemon PID
        char *volume_path;      // Associated volume path
        time_t last_activity;   // Last request timestamp
        int active_connections; // Number of active AFP connections
    } dbd_processes[MAX_VOLUMES];
    
    // Configuration
    char *config_dir;           // Configuration directory
    uid_t db_uid;              // Database daemon user ID
    gid_t db_gid;              // Database daemon group ID
};

2. cnid_dbd - Database Daemon

Each volume has a dedicated database daemon that provides:

Implementation Files:

• etc/cnid_dbd/cnid_dbd.c - Main database daemon implementation
• etc/cnid_dbd/db_param.c - Database parameter management
• etc/cnid_dbd/dbif.c - Database interface layer
• etc/cnid_dbd/dbd.h - Database daemon definitions
• etc/cnid_dbd/pack.c - Data serialization utilities

Core Functions

• CNID Assignment: Generates unique identifiers for files/directories
• Path Resolution: Converts file paths to CNIDs and vice versa
• Database Maintenance: Handles Berkeley DB operations and maintenance
• Transaction Management: Ensures database consistency and recovery

Database Schema

// CNID database record structure
struct cnid_record {
    cnid_t cnid;                // Unique catalog node ID
    cnid_t parent_did;          // Parent directory ID
    char *name;                 // Filename (Mac format)
    dev_t dev;                  // Device number
    ino_t ino;                  // Inode number
    uint32_t type;              // File type
    uint32_t creator;           // File creator
    time_t ctime;               // Creation time
    time_t mtime;               // Modification time
};

3. Berkeley DB Backend

Netatalk uses Berkeley DB for CNID storage with multiple database files:

Implementation Files:

• libatalk/cnid/dbd/cnid_dbd.c - Berkeley DB backend implementation
• libatalk/cnid/dbd/dbif.c - Database interface and transaction management
• etc/cnid_dbd/db_param.c - Berkeley DB configuration parameters
• bin/cnid/cnid_index.c - Database indexing utilities
• bin/cnid/dbd.c - Database maintenance tools

Database Files

• cnid2.db: Main CNID-to-metadata mapping
• devino.db: Device/inode to CNID mapping
• didname.db: Directory ID + name to CNID mapping
• shortname.db: Short name (8.3) to CNID mapping

Database Organization

graph LR
    subgraph "Berkeley DB Files"
        A[cnid2.db<br/>CNID → Metadata]
        B[devino.db<br/>dev:ino → CNID]
        C[didname.db<br/>did:name → CNID]
        D[shortname.db<br/>shortname → CNID]
    end
    
    subgraph "Lookup Patterns"
        E[Path Resolution]
        F[CNID Lookup]
        G[Duplicate Detection]
        H[Legacy Support]
    end
    
    E --> C
    F --> A
    G --> B
    H --> D

Loading

CNID Operations

Implementation Files

• libatalk/cnid/dbd/cnid_dbd_add.c - CNID assignment implementation
• libatalk/cnid/dbd/cnid_dbd_get.c - CNID lookup operations
• libatalk/cnid/dbd/cnid_dbd_lookup.c - Path resolution implementation
• libatalk/cnid/dbd/cnid_dbd_update.c - CNID update operations
• libatalk/cnid/dbd/cnid_dbd_delete.c - CNID deletion operations

1. CNID Assignment Process

sequenceDiagram
    participant AFP as afpd
    participant Meta as cnid_metad
    participant DBD as cnid_dbd
    participant BDB as Berkeley DB
    participant FS as Filesystem
    
    AFP->>Meta: Create file request
    Meta->>DBD: Forward CNID add request
    DBD->>FS: Stat file for dev/ino
    FS-->>DBD: File metadata
    DBD->>BDB: Check for existing CNID
    BDB-->>DBD: No existing record
    DBD->>DBD: Generate new CNID
    DBD->>BDB: Store CNID record
    BDB-->>DBD: Record stored
    DBD-->>Meta: Return new CNID
    Meta-->>AFP: CNID assigned

Loading

2. Path Resolution

// Path to CNID resolution
cnid_t cnid_resolve_path(struct cnid_db *cdb, const char *path) {
    char *component, *ptr;
    cnid_t did = DIRDID_ROOT;  // Start from root directory
    cnid_t cnid;
    
    // Split path into components
    char *path_copy = strdup(path);
    component = strtok_r(path_copy, "/", &ptr);
    
    while (component != NULL) {
        // Look up component in current directory
        cnid = cnid_get(cdb, did, component, strlen(component));
        if (cnid == CNID_INVALID) {
            free(path_copy);
            return CNID_INVALID;
        }
        
        // Update current directory ID
        did = cnid;
        component = strtok_r(NULL, "/", &ptr);
    }
    
    free(path_copy);
    return did;
}

3. Database Lookup Operations

Primary Operations

// Core CNID operations
cnid_t cnid_add(struct cnid_db *cdb, const struct stat *st,
                cnid_t did, const char *name, size_t len, char *hint);

cnid_t cnid_get(struct cnid_db *cdb, cnid_t did, 
                const char *name, size_t len);

cnid_t cnid_lookup(struct cnid_db *cdb, const struct stat *st,
                   cnid_t did, const char *name, size_t len);

int cnid_update(struct cnid_db *cdb, cnid_t id, const struct stat *st,
                cnid_t did, const char *name, size_t len);

int cnid_delete(struct cnid_db *cdb, cnid_t id);

cnid_t cnid_rebuild_add(struct cnid_db *cdb, const struct stat *st,
                        cnid_t did, const char *name, size_t len, cnid_t hint);

Database Query Patterns

// Device/inode lookup for duplicate detection
static cnid_t get_cnid_by_devino(DB *db, dev_t dev, ino_t ino) {
    DBT key, data;
    struct devino_key dkey;
    cnid_t cnid;
    int ret;
    
    memset(&key, 0, sizeof(key));
    memset(&data, 0, sizeof(data));
    
    dkey.dev = dev;
    dkey.ino = ino;
    
    key.data = &dkey;
    key.size = sizeof(dkey);
    data.data = &cnid;
    data.ulen = sizeof(cnid);
    data.flags = DB_DBT_USERMEM;
    
    ret = db->get(db, NULL, &key, &data, 0);
    return (ret == 0) ? cnid : CNID_INVALID;
}

Transaction Management

Implementation Files

• etc/cnid_dbd/dbif.c - Transaction management and database interface
• libatalk/cnid/dbd/cnid_dbd_resolve.c - Transaction-safe path resolution
• etc/cnid_dbd/db_param.c - Transaction configuration parameters
• bin/cnid/dbd_rebuild.c - Database recovery and rebuild utilities

Database Consistency

The CNID system uses Berkeley DB transactions to ensure consistency:

// Transaction wrapper for CNID operations
int cnid_transaction_begin(struct cnid_db *cdb) {
    int ret;
    
    if (cdb->txn != NULL) {
        return -1;  // Transaction already active
    }
    
    ret = cdb->env->txn_begin(cdb->env, NULL, &cdb->txn, 0);
    if (ret != 0) {
        LOG(log_error, "Failed to begin transaction: %s", db_strerror(ret));
        return -1;
    }
    
    return 0;
}

int cnid_transaction_commit(struct cnid_db *cdb) {
    int ret;
    
    if (cdb->txn == NULL) {
        return -1;  // No active transaction
    }
    
    ret = cdb->txn->commit(cdb->txn, 0);
    cdb->txn = NULL;
    
    if (ret != 0) {
        LOG(log_error, "Failed to commit transaction: %s", db_strerror(ret));
        return -1;
    }
    
    return 0;
}

Recovery and Repair

// Database recovery operations
int cnid_recover_database(const char *db_dir) {
    DB_ENV *env;
    int ret;
    
    // Create database environment
    ret = db_env_create(&env, 0);
    if (ret != 0) {
        return -1;
    }
    
    // Run automatic recovery
    ret = env->open(env, db_dir, 
                    DB_CREATE | DB_INIT_MPOOL | DB_INIT_TXN | 
                    DB_INIT_LOG | DB_RECOVER, 0);
    
    if (ret != 0) {
        LOG(log_error, "Database recovery failed: %s", db_strerror(ret));
        env->close(env, 0);
        return -1;
    }
    
    env->close(env, 0);
    return 0;
}

Performance Optimizations

1. Caching Strategies

CNID Cache

// CNID lookup cache
struct cnid_cache_entry {
    cnid_t cnid;
    cnid_t parent_did;
    char *name;
    time_t access_time;
    struct cnid_cache_entry *next;
};

// Cache implementation
static struct cnid_cache_entry *cnid_cache[CNID_CACHE_SIZE];

static cnid_t cnid_cache_lookup(cnid_t did, const char *name) {
    unsigned int hash = hash_function(did, name) % CNID_CACHE_SIZE;
    struct cnid_cache_entry *entry = cnid_cache[hash];
    
    while (entry) {
        if (entry->parent_did == did && strcmp(entry->name, name) == 0) {
            entry->access_time = time(NULL);  // Update LRU
            return entry->cnid;
        }
        entry = entry->next;
    }
    
    return CNID_INVALID;
}

2. Database Tuning

Berkeley DB Configuration

// Database environment setup
static int setup_db_environment(DB_ENV **env, const char *db_dir) {
    DB_ENV *dbenv;
    int ret;
    
    ret = db_env_create(&dbenv, 0);
    if (ret != 0) return ret;
    
    // Set cache size (64MB)
    ret = dbenv->set_cachesize(dbenv, 0, 64 * 1024 * 1024, 1);
    if (ret != 0) goto error;
    
    // Set log buffer size
    ret = dbenv->set_lg_bsize(dbenv, 1024 * 1024);
    if (ret != 0) goto error;
    
    // Set lock detection
    ret = dbenv->set_lk_detect(dbenv, DB_LOCK_DEFAULT);
    if (ret != 0) goto error;
    
    // Open environment
    ret = dbenv->open(dbenv, db_dir,
                      DB_CREATE | DB_INIT_MPOOL | DB_INIT_LOCK |
                      DB_INIT_LOG | DB_INIT_TXN | DB_RECOVER,
                      0);
    
    if (ret == 0) {
        *env = dbenv;
        return 0;
    }
    
error:
    dbenv->close(dbenv, 0);
    return ret;
}

Integration with AFP Operations

File Creation Flow

sequenceDiagram
    participant Client as Mac Client
    participant AFP as afpd
    participant CNID as CNID System
    participant FS as Filesystem
    
    Client->>AFP: FPCreateFile
    AFP->>FS: Create file on filesystem
    FS-->>AFP: File created (dev/ino)
    AFP->>CNID: cnid_add(dev, ino, parent_did, name)
    CNID->>CNID: Generate unique CNID
    CNID->>CNID: Store CNID record
    CNID-->>AFP: Return new CNID
    AFP-->>Client: File created with CNID

Loading

File Movement Handling

// Handle file rename/move operations
int cnid_move_file(struct cnid_db *cdb, cnid_t cnid, 
                   cnid_t new_did, const char *new_name) {
    struct cnid_record rec;
    int ret;
    
    // Begin transaction
    cnid_transaction_begin(cdb);
    
    // Get current record
    ret = cnid_get_record(cdb, cnid, &rec);
    if (ret != 0) {
        cnid_transaction_abort(cdb);
        return ret;
    }
    
    // Update record with new location
    rec.parent_did = new_did;
    free(rec.name);
    rec.name = strdup(new_name);
    
    // Store updated record
    ret = cnid_update_record(cdb, cnid, &rec);
    if (ret != 0) {
        cnid_transaction_abort(cdb);
        cnid_free_record(&rec);
        return ret;
    }
    
    // Commit transaction
    ret = cnid_transaction_commit(cdb);
    cnid_free_record(&rec);
    
    return ret;
}

Database Maintenance

1. Database Compaction

// Periodic database compaction
int cnid_compact_database(struct cnid_db *cdb) {
    DB_COMPACT c_data;
    int ret, i;
    
    memset(&c_data, 0, sizeof(c_data));
    
    // Compact each database file
    DB *databases[] = {cdb->cnid_db, cdb->devino_db, 
                       cdb->didname_db, cdb->shortname_db};
    
    for (i = 0; i < 4; i++) {
        if (databases[i] != NULL) {
            ret = databases[i]->compact(databases[i], NULL, NULL, NULL, 
                                        &c_data, DB_FREE_SPACE, NULL);
            if (ret != 0) {
                LOG(log_warning, "Database compaction failed: %s", 
                    db_strerror(ret));
            } else {
                LOG(log_info, "Compacted database, freed %u pages", 
                    c_data.compact_pages_free);
            }
        }
    }
    
    return 0;
}

2. Consistency Checking

// Verify database consistency
int cnid_verify_database(const char *db_dir) {
    DB *db;
    int ret;
    char db_file[PATH_MAX];
    
    const char *db_files[] = {"cnid2.db", "devino.db", 
                              "didname.db", "shortname.db"};
    
    for (int i = 0; i < 4; i++) {
        snprintf(db_file, sizeof(db_file), "%s/%s", db_dir, db_files[i]);
        
        ret = db_create(&db, NULL, 0);
        if (ret != 0) continue;
        
        ret = db->verify(db, db_file, NULL, NULL, 0);
        db->close(db, 0);
        
        if (ret != 0) {
            LOG(log_error, "Database verification failed for %s: %s", 
                db_files[i], db_strerror(ret));
            return ret;
        }
    }
    
    LOG(log_info, "Database verification completed successfully");
    return 0;
}

Configuration and Tuning

Database Parameters

// CNID database configuration
struct cnid_config {
    char *db_dir;               // Database directory
    size_t cache_size;          // Memory cache size
    int sync_mode;              // Synchronization mode
    int deadlock_detection;     // Deadlock detection method
    int checkpoint_interval;    // Automatic checkpoint interval
    int log_autoremove;         // Automatic log file removal
    
    // Performance tuning
    int page_size;              // Database page size
    int lock_timeout;           // Lock timeout (microseconds)
    int txn_timeout;            // Transaction timeout (microseconds)
};

The CNID system provides the foundation for reliable file identification in Netatalk, ensuring that Mac clients can maintain stable references to files and directories even as they are renamed, moved, or modified on the server filesystem. Its robust Berkeley DB backend and transaction-based architecture provide excellent reliability and performance characteristics.