Decompollaborate · AngheloAlf · Jun 2, 2025 · Jun 2, 2025 · Jun 2, 2025 · Jun 2, 2025
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -79,28 +79,28 @@ jobs:
         run: python3 -m pip install .
 
       - name: bss_check
-        run: python3 -m mapfile_parser bss_check tests/maps/drmario64.us.map tests/maps/drmario64.us.map
+        run: python3 -m mapfile_parser bss_check tests/maps/gnuld/n64/drmario64.us.map tests/maps/gnuld/n64/drmario64.us.map
 
       - name: first_diff
-        run: python3 -m mapfile_parser first_diff tests/maps/drmario64.us.map tests/maps/drmario64.us.map tests/maps/drmario64.us.map tests/maps/drmario64.us.map
+        run: python3 -m mapfile_parser first_diff tests/maps/gnuld/n64/drmario64.us.map tests/maps/gnuld/n64/drmario64.us.map tests/maps/gnuld/n64/drmario64.us.map tests/maps/gnuld/n64/drmario64.us.map
 
       - name: jsonify
-        run: python3 -m mapfile_parser jsonify tests/maps/drmario64.us.map
+        run: python3 -m mapfile_parser jsonify tests/maps/gnuld/n64/drmario64.us.map
 
       - name: objdiff_report
-        run: python3 -m mapfile_parser objdiff_report tests/maps/drmario64.us.map objdiff_report.json
+        run: python3 -m mapfile_parser objdiff_report tests/maps/gnuld/n64/drmario64.us.map objdiff_report.json
 
       - name: pj64_syms
-        run: python3 -m mapfile_parser pj64_syms tests/maps/drmario64.us.map
+        run: python3 -m mapfile_parser pj64_syms tests/maps/gnuld/n64/drmario64.us.map
 
       - name: progress
-        run: python3 -m mapfile_parser progress tests/maps/drmario64.us.map asm asm/nonmatchings
+        run: python3 -m mapfile_parser progress tests/maps/gnuld/n64/drmario64.us.map asm asm/nonmatchings
 
       - name: sym_info
-        run: python3 -m mapfile_parser sym_info tests/maps/drmario64.us.map entrypoint
+        run: python3 -m mapfile_parser sym_info tests/maps/gnuld/n64/drmario64.us.map entrypoint
 
       - name: symbol_sizes_csv
-        run: python3 -m mapfile_parser symbol_sizes_csv tests/maps/drmario64.us.map
+        run: python3 -m mapfile_parser symbol_sizes_csv tests/maps/gnuld/n64/drmario64.us.map
 
       - name: upload_frogress
-        run: python3 -m mapfile_parser upload_frogress tests/maps/drmario64.us.map asm asm/nonmatchings drmario64 us code --verbose --dry-run
+        run: python3 -m mapfile_parser upload_frogress tests/maps/gnuld/n64/drmario64.us.map asm asm/nonmatchings drmario64 us code --verbose --dry-run
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [2.9.4] - 2025-06-02
+
+### Changed
+
+- Implement parsing for `mwld` 2.7+ generated mapfiles.
+- Handle `*fill*`s in `mwld`.
+
 ## [2.9.3] - 2025-06-01
 
 ### Fixed
@@ -613,6 +620,7 @@ Full changes: <https://github.com/Decompollaborate/mapfile_parser/compare/702a73
 - Initial release
 
 [unreleased]: https://github.com/Decompollaborate/mapfile_parser/compare/master...develop
+[2.9.4]: https://github.com/Decompollaborate/mapfile_parser/compare/2.9.3...2.9.4
 [2.9.3]: https://github.com/Decompollaborate/mapfile_parser/compare/2.9.2...2.9.3
 [2.9.2]: https://github.com/Decompollaborate/mapfile_parser/compare/2.9.1...2.9.2
 [2.9.1]: https://github.com/Decompollaborate/mapfile_parser/compare/2.9.0...2.9.1

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -3,7 +3,7 @@
 
 [package]
 name = "mapfile_parser"
-version = "2.9.3"
+version = "2.9.4"
 edition = "2021"
 rust-version = "1.74.0"
 authors = ["Anghelo Carvajal <[email protected]>"]

diff --git a/README.md b/README.md
@@ -36,7 +36,7 @@ If you use a `requirements.txt` file in your repository, then you can add
 this library with the following line:
 
 ```txt
-mapfile_parser>=2.9.3,<3.0.0
+mapfile_parser>=2.9.4,<3.0.0
 ```
 
 #### Development version
@@ -75,7 +75,7 @@ cargo add mapfile_parser
 Or add the following line manually to your `Cargo.toml` file:
 
 ```toml
-mapfile_parser = "2.9.3"
+mapfile_parser = "2.9.4"
 ```
 
 ## Versioning and changelog

diff --git a/pyproject.toml b/pyproject.toml
@@ -3,7 +3,7 @@
 
 [project]
 name = "mapfile_parser"
-version = "2.9.3"
+version = "2.9.4"
 description = "Map file parser library focusing decompilation projects"
 readme = "README.md"
 requires-python = ">=3.9"

diff --git a/src/mapfile_parser/__init__.py b/src/mapfile_parser/__init__.py
@@ -5,7 +5,7 @@
 
 from __future__ import annotations
 
-__version_info__ = (2, 9, 3)
+__version_info__ = (2, 9, 4)
 __version__ = ".".join(map(str, __version_info__)) # + "-dev0"
 __author__ = "Decompollaborate"
 

diff --git a/src/rs/lib.rs b/src/rs/lib.rs
@@ -63,6 +63,6 @@ mod tests {
 
     #[test]
     fn w0_000_map() {
-        let _ = MapFile::new_from_map_file(&PathBuf::from("tests/maps/w0_000.map"));
+        let _ = MapFile::new_from_map_file(&PathBuf::from("tests/maps/gnuld/misc/w0_000.map"));
     }
 }
diff --git a/src/rs/parser.rs b/src/rs/parser.rs
@@ -232,7 +232,7 @@ impl MapFile {
                                     filepath,
                                     vram,
                                     size,
-                                    section_type,
+                                    section_type.into(),
                                 ));
                         } else if let Some(section_alone_match) =
                             regex_section_alone_entry.captures(prev_line)
@@ -250,7 +250,7 @@ impl MapFile {
                                     filepath,
                                     vram,
                                     size,
-                                    section_type,
+                                    section_type.into(),
                                 ));
                         }
                     }
@@ -319,7 +319,7 @@ impl MapFile {
                             filepath,
                             vram,
                             size,
-                            &section_type,
+                            section_type,
                         ));
                 }
             }
@@ -497,8 +497,8 @@ impl MapFile {
                     }
 
                     let mut new_section =
-                        section::Section::new_fill(filepath, vram, size, &section_type);
-                    if !utils::is_noload_section(&section_type) {
+                        section::Section::new_fill(filepath, vram, size, section_type);
+                    if !utils::is_noload_section(&new_section.section_type) {
                         new_section.vrom = vrom;
                     }
                     current_segment.sections_list.push(new_section);
@@ -510,9 +510,13 @@ impl MapFile {
                     if size > 0 {
                         let current_segment = temp_segment_list.last_mut().unwrap();
 
-                        let mut new_section =
-                            section::Section::new_default(filepath, vram, size, section_type);
-                        if !utils::is_noload_section(section_type) {
+                        let mut new_section = section::Section::new_default(
+                            filepath,
+                            vram,
+                            size,
+                            section_type.into(),
+                        );
+                        if !utils::is_noload_section(&new_section.section_type) {
                             new_section.vrom = vrom;
                         }
                         new_section.align = Some(align);
@@ -608,34 +612,41 @@ impl MapFile {
         let map_data = preprocess_map_data_mw(map_contents);
 
         let memory_map = parse_memory_map_mw(map_data);
-
-        // Almost every line starts with this information, so instead of duplicating it we put them on one single regex
-        let regex_row_entry = Regex::new(r"^\s*(?P<starting>[0-9a-fA-F]+)\s+(?P<size>[0-9a-fA-F]+)\s+(?P<vram>[0-9a-fA-F]+)\s+(?P<align>[0-9a-fA-F]+)\s+(?P<subline>.+)").unwrap();
-
-        let regex_segment_entry = Regex::new(r"^(?P<name>.+) section layout$").unwrap();
-        let regex_label_entry =
-            Regex::new(r"^(?P<label>lbl_[0-9A-F]{8})\s+(?P<filename>.+?)\s*$").unwrap();
-        let regex_symbol_entry =
-            Regex::new(r"^\s*(?P<name>[^ ]+)\s+(?P<filename>.+?)\s*$").unwrap();
+        let regex_entries = MwRegexEntries::new(map_data);
 
         let mut temp_segment_list = vec![segment::Segment::new_placeholder()];
 
         // Use a bunch of characters that shouldn't be valid in any os as a marker that we haven't found a file yet.
-        let mut current_filename = "invalid file <>:\"/\\|?*".to_string();
+        let invalid_file_name = "invalid file <>:\"/\\|?*";
+        let mut current_filename = invalid_file_name.to_string();
 
         for line in map_data.lines() {
-            // Check for regex_row_entry since it is more likely to match
-            if let Some(row_entry_match) = regex_row_entry.captures(line) {
+            // Check for regex_entries.common_row first since it is more likely to match
+            if let (Some(row_entry_match), false) = (
+                regex_entries.common_row.captures(line),
+                temp_segment_list.is_empty(),
+            ) {
                 let starting = utils::parse_hex(&row_entry_match["starting"]);
                 let size = utils::parse_hex(&row_entry_match["size"]);
                 let vram = utils::parse_hex(&row_entry_match["vram"]);
                 let align = utils::parse_hex(&row_entry_match["align"]);
 
+                let rom = row_entry_match.name("rom").map_or_else(
+                    || {
+                        temp_segment_list
+                            .last()
+                            .unwrap()
+                            .vrom
+                            .map(|segment_rom| segment_rom + starting)
+                    },
+                    |x| Some(utils::parse_hex(x.as_str())),
+                );
+
                 let subline = &row_entry_match["subline"];
 
-                if regex_label_entry.is_match(subline) {
+                if regex_entries.label.is_match(subline) {
                     // pass
-                } else if let Some(symbol_entry_match) = regex_symbol_entry.captures(subline) {
+                } else if let Some(symbol_entry_match) = regex_entries.symbol.captures(subline) {
                     let filename = &symbol_entry_match["filename"];
 
                     if filename == current_filename {
@@ -652,7 +663,7 @@ impl MapFile {
                                 new_symbol.size = size;
                             }
                             if !current_section.is_noload_section() {
-                                new_symbol.vrom = current_segment.vrom.map(|x| x + starting)
+                                new_symbol.vrom = rom
                             }
                             if align > 0 {
                                 new_symbol.align = Some(align);
@@ -662,27 +673,62 @@ impl MapFile {
                         }
                     } else {
                         // New file!
-                        current_filename = filename.to_string();
-
                         if size > 0 {
                             let section_type = &symbol_entry_match["name"];
                             let filepath = PathBuf::from(filename);
 
                             let current_segment = temp_segment_list.last_mut().unwrap();
 
-                            let mut new_section =
-                                section::Section::new_default(filepath, vram, size, section_type);
-                            if !utils::is_noload_section(section_type) {
-                                new_section.vrom = current_segment.vrom.map(|x| x + starting)
+                            let mut new_section = section::Section::new_default(
+                                filepath,
+                                vram,
+                                size,
+                                section_type.into(),
+                            );
+                            if !utils::is_noload_section(&new_section.section_type) {
+                                new_section.vrom = rom
                             }
 
                             current_segment.sections_list.push(new_section);
+
+                            // I'm not sure how to treat these cases.
+                            // I guess we can treat them as files without symbols for now...
+                            current_filename = if filename == "Linker Generated Symbol File" {
+                                invalid_file_name.to_string()
+                            } else {
+                                filename.to_string()
+                            };
                         }
                     }
+                } else if regex_entries.fill.is_match(subline) {
+                    // Make a dummy section to handle pads
+                    let current_segment = temp_segment_list.last_mut().unwrap();
+
+                    let mut filepath = PathBuf::new();
+                    let mut section_type = "".to_owned();
+
+                    if let Some(prev_section) = current_segment.sections_list.last() {
+                        let mut name = prev_section.filepath.file_name().unwrap().to_owned();
+
+                        name.push("__fill__");
+                        filepath = prev_section.filepath.with_file_name(name);
+                        section_type.clone_from(&prev_section.section_type);
+                    }
+
+                    let mut new_section =
+                        section::Section::new_fill(filepath, vram, size, section_type);
+                    new_section.align = Some(align);
+                    if !utils::is_noload_section(&new_section.section_type) {
+                        new_section.vrom = rom;
+                    }
+                    current_segment.sections_list.push(new_section);
+
+                    // Don't count this as a valid file.
+                    current_filename = invalid_file_name.to_string();
                 } else {
-                    println!("{}", subline);
+                    // println!("'{}'", subline);
                 }
-            } else if let Some(segment_entry_match) = regex_segment_entry.captures(line) {
+            } else if let Some(segment_entry_match) = regex_entries.segment.captures(line) {
                 let name = &segment_entry_match["name"];
 
                 let new_segment = if let Some(segment_entry) = memory_map.get(name) {
@@ -697,14 +743,17 @@ impl MapFile {
                 };
 
                 temp_segment_list.push(new_segment);
+
+                // Reset the tracked filename state.
+                // This avoid carrying the filename from one segment to the other
+                current_filename = invalid_file_name.to_string();
             }
         }
 
         self.segments_list = Self::post_process_segments_mw(temp_segment_list);
     }
 
     fn post_process_segments_mw(temp_segment_list: Vec<segment::Segment>) -> Vec<segment::Segment> {
-        // TODO: actually implement
         let mut segments_list = Vec::with_capacity(temp_segment_list.len());
 
         for (i, segment) in temp_segment_list.into_iter().enumerate() {
@@ -793,6 +842,40 @@ fn parse_memory_map_mw(map_data: &str) -> HashMap<String, MwMemoryMapEntry> {
     memory_map
 }
 
+struct MwRegexEntries {
+    common_row: Regex,
+    segment: Regex,
+    label: Regex,
+    symbol: Regex,
+    fill: Regex,
+}
+
+impl MwRegexEntries {
+    fn new(map_data: &str) -> Self {
+        // Almost every line starts with this information, so instead of duplicating it we put them on one single regex
+        let common_row = if map_data.contains("address  Size   address  offset") {
+            // mwld 2.7+
+            Regex::new(r"^\s*(?P<starting>[0-9a-fA-F]+)\s+(?P<size>[0-9a-fA-F]+)\s+(?P<vram>[0-9a-fA-F]+)\s+(?P<rom>[0-9a-fA-F]+)\s+(?P<align>[0-9a-fA-F]+)\s+(?P<subline>.+)").unwrap()
+        } else {
+            // mwld 1.3.2-
+            Regex::new(r"^\s*(?P<starting>[0-9a-fA-F]+)\s+(?P<size>[0-9a-fA-F]+)\s+(?P<vram>[0-9a-fA-F]+)\s+(?P<align>[0-9a-fA-F]+)\s+(?P<subline>.+)").unwrap()
+        };
+
+        let segment = Regex::new(r"^(?P<name>.+) section layout$").unwrap();
+        let label = Regex::new(r"^(?P<label>lbl_[0-9A-F]{8})\s+(?P<filename>.+?)\s*$").unwrap();
+        let symbol = Regex::new(r"^\s*(?P<name>[^ ]+)\s+(?P<filename>.+?)\s*$").unwrap();
+        let fill = Regex::new(r"^\s*\*fill\*\s*$").unwrap();
+
+        Self {
+            common_row,
+            segment,
+            label,
+            symbol,
+            fill,
+        }
+    }
+}
+
 impl MapFile {
     fn preprocess_map_data_gnu(map_data: &str) -> &str {
         // Skip the stuff we don't care about