From c8ea2438c22a2b9ef920e6aeb3a9f5d30e95ee97 Mon Sep 17 00:00:00 2001
From: Alan O'Cais <a.ocais@fz-juelich.de>
Date: Fri, 22 Feb 2019 17:14:30 +0100
Subject: [PATCH 1/8] Tolerate existing edges in digraph

---
 easybuild/framework/easyconfig/tools.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py
index ba7b634b6d..c28e9f400f 100644
--- a/easybuild/framework/easyconfig/tools.py
+++ b/easybuild/framework/easyconfig/tools.py
@@ -206,7 +206,13 @@ def mk_node_name(spec):
     edge_attrs = [('style', 'dotted'), ('color', 'blue'), ('arrowhead', 'diamond')]
     for spec in specs:
         for dep in spec['ec'].all_dependencies:
-            dgr.add_edge((spec['module'], dep))
+            try:
+                dgr.add_edge((spec['module'], dep))
+            except Exception as e:
+                if "already in digraph" in str(e):
+                    continue
+                else:
+                    raise e
             if dep in spec['ec'].build_dependencies:
                 dgr.add_edge_attributes((spec['module'], dep), attrs=edge_attrs)
 

From 70f99cc640558faf4b650299dd386f4fd3c80f84 Mon Sep 17 00:00:00 2001
From: Alan O'Cais <a.ocais@fz-juelich.de>
Date: Tue, 4 Jun 2019 10:23:43 +0200
Subject: [PATCH 2/8] Add explanatory comment and logging

---
 easybuild/framework/easyconfig/tools.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py
index b7e2460fab..98dfd7b8cb 100644
--- a/easybuild/framework/easyconfig/tools.py
+++ b/easybuild/framework/easyconfig/tools.py
@@ -208,11 +208,15 @@ def mk_node_name(spec):
         for dep in spec['ec'].all_dependencies:
             try:
                 dgr.add_edge((spec['module'], dep))
-            except Exception as e:
-                if "already in digraph" in str(e):
+            except Exception as err:
+                # When multiple specs are given we can easily have overlapping edges (i.e., common dependency trees)
+                # so we need to explicitly account this in our digraph
+                if "already in digraph" in str(err):
+                    _log.info("Edge (%s) -> (%s) already exists in digraph (but duplication is permitted)" %
+                              (spec['module'], dep))
                     continue
                 else:
-                    raise e
+                    raise err
             if dep in spec['ec'].build_dependencies:
                 dgr.add_edge_attributes((spec['module'], dep), attrs=edge_attrs)
 

From 19d797e58ce12e82b240d49c2dbd46e804e07c17 Mon Sep 17 00:00:00 2001
From: Alan O'Cais <a.ocais@fz-juelich.de>
Date: Tue, 4 Jun 2019 10:34:33 +0200
Subject: [PATCH 3/8] Add script that extracts node from large dotfile

---
 .../scripts/extract_node_from_dotfile.sh      | 50 +++++++++++++++++++
 1 file changed, 50 insertions(+)
 create mode 100644 easybuild/scripts/extract_node_from_dotfile.sh

diff --git a/easybuild/scripts/extract_node_from_dotfile.sh b/easybuild/scripts/extract_node_from_dotfile.sh
new file mode 100644
index 0000000000..c047416f72
--- /dev/null
+++ b/easybuild/scripts/extract_node_from_dotfile.sh
@@ -0,0 +1,50 @@
+#!/bin/bash
+
+find_children () {
+ # The fact that we put the semicolon straight after means we exclude anywhere the module
+ # is only required as a build dependency (since these cases have extra formatting the
+ # semicolon comes later);
+ # the sed command makes sure we ignore whether the module is hidden or not
+ grep ' "'$3'";' $1 | sed s#/\\.#/#g >> $2
+ grep ' "'$3'";' $1 | awk '{print $1}'| xargs -i bash -c "map_dep $1 $2 {}"
+}
+
+map_dep () {
+ # Search for a node that matches the string and add a comment marker at the end
+ # to leverage with grep later
+ # (if non-installed software had additional formatting they would be ignored)
+ # the sed command makes sure we ignore whether the module is hidden or not
+ grep "^"'"'$3'";' $1 | awk '{print $1" // xxnodexx"}' | sed s#/\\.#/#g >> $2
+ if [ $? ]
+ then
+   find_children $1 $2 $3
+ fi
+}
+
+export -f find_children
+export -f map_dep
+
+# Check command line
+if [ "$#" -ne 3 ]; then
+    echo -e "Expected command line:\n\t<script> <input dot file> <output dot file> <node to search for>"
+    exit 1
+fi
+
+# Begin digraph in output file
+echo digraph graphname \{ > $2
+
+# Use a temporary file to store nodes and edges
+> temp.dot
+# Gather nodes and edges related to $3
+map_dep $1 temp.dot $3
+
+# There is potential duplication due to nodes being followed multiple times so let's
+# remove it:
+# put nodes first (we used a marker to identify them), make sure they are unique
+cat temp.dot | sort | uniq | grep xxnodexx >> $2
+# Then edges, also make sure they are unique
+cat temp.dot | sort | uniq | grep -v xxnodexx >> $2
+
+# Clean up
+rm temp.dot
+echo \} >> $2
\ No newline at end of file

From 5ac33e7ea16e2f0358c16809138f62f8bc9d6a73 Mon Sep 17 00:00:00 2001
From: Alan O'Cais <a.ocais@fz-juelich.de>
Date: Tue, 4 Jun 2019 10:36:14 +0200
Subject: [PATCH 4/8] Add execution permission to new script

---
 easybuild/scripts/extract_node_from_dotfile.sh | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 mode change 100644 => 100755 easybuild/scripts/extract_node_from_dotfile.sh

diff --git a/easybuild/scripts/extract_node_from_dotfile.sh b/easybuild/scripts/extract_node_from_dotfile.sh
old mode 100644
new mode 100755

From 556428eb2289d5a5f0cc558f9e496e4f5373c148 Mon Sep 17 00:00:00 2001
From: Alan O'Cais <a.ocais@fz-juelich.de>
Date: Wed, 12 Jun 2019 17:38:36 +0200
Subject: [PATCH 5/8] Address comments

---
 easybuild/framework/easyconfig/tools.py       |  5 +-
 .../scripts/extract_node_from_dotfile.sh      | 98 ++++++++++++++-----
 2 files changed, 78 insertions(+), 25 deletions(-)

diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py
index 98dfd7b8cb..ddd82c601b 100644
--- a/easybuild/framework/easyconfig/tools.py
+++ b/easybuild/framework/easyconfig/tools.py
@@ -70,6 +70,7 @@
     # PyGraph (used for generating dependency graphs)
     # https://pypi.python.org/pypi/python-graph-core
     from pygraph.classes.digraph import digraph
+    from pygraph.classes.exceptions import AdditionError
     # https://pypi.python.org/pypi/python-graph-dot
     import pygraph.readwrite.dot as dot
     # graphviz (used for creating dependency graph images)
@@ -208,11 +209,11 @@ def mk_node_name(spec):
         for dep in spec['ec'].all_dependencies:
             try:
                 dgr.add_edge((spec['module'], dep))
-            except Exception as err:
+            except AdditionError as err:
                 # When multiple specs are given we can easily have overlapping edges (i.e., common dependency trees)
                 # so we need to explicitly account this in our digraph
                 if "already in digraph" in str(err):
-                    _log.info("Edge (%s) -> (%s) already exists in digraph (but duplication is permitted)" %
+                    _log.info("Edge (%s) -> (%s) already exists in digraph (but duplication is permitted)",
                               (spec['module'], dep))
                     continue
                 else:
diff --git a/easybuild/scripts/extract_node_from_dotfile.sh b/easybuild/scripts/extract_node_from_dotfile.sh
index c047416f72..8afcb00707 100755
--- a/easybuild/scripts/extract_node_from_dotfile.sh
+++ b/easybuild/scripts/extract_node_from_dotfile.sh
@@ -1,24 +1,69 @@
 #!/bin/bash
+##
+# Copyright 2012-2019 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild.  If not, see <http://www.gnu.org/licenses/>.
+##
+
+# @author: Alan O'Cais (Juelich Supercomputing Centre)
+
+##
+# Use case:
+#
+# A large dotfile that contains all software installed with EasyBuild can be generated with a command line such as:
+#   eb --dep-graph=depgraph.dot $EASYBUILD_INSTALLPATH/software/*/*/easybuild/*.eb
+# Such a dotfile is (typically) too large to deal with directly. This script can extract a particular node from such a
+# dotfile, e.g.,
+#   ./<script> <input dot file> <output dot file> "Compiler/GCCcore/7.3.0/h5py/2.8.0-serial-Python-3.6.6"
+# which can be used to help evaluate the impact of uninstalling that particular software package
+##
 
 find_children () {
- # The fact that we put the semicolon straight after means we exclude anywhere the module
- # is only required as a build dependency (since these cases have extra formatting the
- # semicolon comes later);
- # the sed command makes sure we ignore whether the module is hidden or not
- grep ' "'$3'";' $1 | sed s#/\\.#/#g >> $2
- grep ' "'$3'";' $1 | awk '{print $1}'| xargs -i bash -c "map_dep $1 $2 {}"
+    input_dotfile=$1
+    output_dotfile=$2
+    node_to_search_for=$3
+    # The fact that we put the semicolon straight after means we exclude anywhere the module
+    # is only required as a build dependency (since these cases have extra formatting the
+    # semicolon comes later);
+    # the sed command makes sure we ignore whether the module is hidden or not
+    grep ' "'${node_to_search_for}'";' ${input_dotfile} | sed s#/\\.#/#g >> ${output_dotfile}
+    grep ' "'${node_to_search_for}'";' ${input_dotfile} | awk '{print $1}'| \
+        xargs -i bash -c "map_dep ${input_dotfile} ${output_dotfile} {}"
 }
 
 map_dep () {
- # Search for a node that matches the string and add a comment marker at the end
- # to leverage with grep later
- # (if non-installed software had additional formatting they would be ignored)
- # the sed command makes sure we ignore whether the module is hidden or not
- grep "^"'"'$3'";' $1 | awk '{print $1" // xxnodexx"}' | sed s#/\\.#/#g >> $2
- if [ $? ]
- then
-   find_children $1 $2 $3
- fi
+    input_dotfile=$1
+    output_dotfile=$2
+    node_to_search_for=$3
+    # Search for a node that matches the string and add a comment marker at the end
+    # to leverage with grep later
+    # (if non-installed software had additional formatting they would be ignored)
+    # the sed command makes sure we ignore whether the module is hidden or not
+    grep "^"'"'${node_to_search_for}'";' ${input_dotfile} | awk '{print $1" // xxnodexx"}' | \
+        sed s#/\\.#/#g >> ${output_dotfile}
+    if [ $? ]
+    then
+        find_children ${input_dotfile} ${output_dot_file} ${node_to_search_for}
+    fi
 }
 
 export -f find_children
@@ -30,21 +75,28 @@ if [ "$#" -ne 3 ]; then
     exit 1
 fi
 
+input_dotfile=$1
+output_dotfile=$2
+node_to_search_for=$3
+
 # Begin digraph in output file
-echo digraph graphname \{ > $2
+echo digraph graphname \{ > ${output_dotfile}
 
 # Use a temporary file to store nodes and edges
-> temp.dot
-# Gather nodes and edges related to $3
-map_dep $1 temp.dot $3
+tmpfile=`mktemp`
+# initialise the file
+> ${tmpfile}
+
+# Gather nodes and edges related to ${node_to_search_for}
+map_dep ${input_dotfile} ${tmpfile} ${node_to_search_for}
 
 # There is potential duplication due to nodes being followed multiple times so let's
 # remove it:
 # put nodes first (we used a marker to identify them), make sure they are unique
-cat temp.dot | sort | uniq | grep xxnodexx >> $2
+cat ${tmpfile} | sort | uniq | grep xxnodexx >> ${output_dotfile}
 # Then edges, also make sure they are unique
-cat temp.dot | sort | uniq | grep -v xxnodexx >> $2
+cat ${tmpfile} | sort | uniq | grep -v xxnodexx >> ${output_dotfile}
 
 # Clean up
-rm temp.dot
-echo \} >> $2
\ No newline at end of file
+rm ${tmpfile}
+echo \} >> ${output_dotfile}
\ No newline at end of file

From 601f586976f9ca08404e1ea2490248baee68166e Mon Sep 17 00:00:00 2001
From: ocaisa <alan.ocais@epfl.ch>
Date: Mon, 31 Jan 2022 17:19:32 +0100
Subject: [PATCH 6/8] Update extract_node_from_dotfile.sh

---
 easybuild/scripts/extract_node_from_dotfile.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/easybuild/scripts/extract_node_from_dotfile.sh b/easybuild/scripts/extract_node_from_dotfile.sh
index 8afcb00707..573ff6f3d4 100755
--- a/easybuild/scripts/extract_node_from_dotfile.sh
+++ b/easybuild/scripts/extract_node_from_dotfile.sh
@@ -62,7 +62,7 @@ map_dep () {
         sed s#/\\.#/#g >> ${output_dotfile}
     if [ $? ]
     then
-        find_children ${input_dotfile} ${output_dot_file} ${node_to_search_for}
+        find_children ${input_dotfile} ${output_dotfile} ${node_to_search_for}
     fi
 }
 
@@ -99,4 +99,4 @@ cat ${tmpfile} | sort | uniq | grep -v xxnodexx >> ${output_dotfile}
 
 # Clean up
 rm ${tmpfile}
-echo \} >> ${output_dotfile}
\ No newline at end of file
+echo \} >> ${output_dotfile}

From 7c86f8eb0c08ce16bfe2aa07f6aa6a204f9db665 Mon Sep 17 00:00:00 2001
From: ocaisa <alan.ocais@epfl.ch>
Date: Mon, 31 Jan 2022 17:41:53 +0100
Subject: [PATCH 7/8] Update extract_node_from_dotfile.sh

---
 easybuild/scripts/extract_node_from_dotfile.sh | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/easybuild/scripts/extract_node_from_dotfile.sh b/easybuild/scripts/extract_node_from_dotfile.sh
index 573ff6f3d4..a44454f556 100755
--- a/easybuild/scripts/extract_node_from_dotfile.sh
+++ b/easybuild/scripts/extract_node_from_dotfile.sh
@@ -38,9 +38,9 @@
 ##
 
 find_children () {
-    input_dotfile=$1
-    output_dotfile=$2
-    node_to_search_for=$3
+    local input_dotfile=$1
+    local output_dotfile=$2
+    local node_to_search_for=$3
     # The fact that we put the semicolon straight after means we exclude anywhere the module
     # is only required as a build dependency (since these cases have extra formatting the
     # semicolon comes later);
@@ -51,9 +51,9 @@ find_children () {
 }
 
 map_dep () {
-    input_dotfile=$1
-    output_dotfile=$2
-    node_to_search_for=$3
+    local input_dotfile=$1
+    local output_dotfile=$2
+    local node_to_search_for=$3
     # Search for a node that matches the string and add a comment marker at the end
     # to leverage with grep later
     # (if non-installed software had additional formatting they would be ignored)
@@ -75,9 +75,9 @@ if [ "$#" -ne 3 ]; then
     exit 1
 fi
 
-input_dotfile=$1
-output_dotfile=$2
-node_to_search_for=$3
+local input_dotfile=$1
+local output_dotfile=$2
+local node_to_search_for=$3
 
 # Begin digraph in output file
 echo digraph graphname \{ > ${output_dotfile}

From d2852a63863d2623ea0cd9075c420080eecc60bb Mon Sep 17 00:00:00 2001
From: ocaisa <alan.ocais@epfl.ch>
Date: Mon, 31 Jan 2022 17:43:12 +0100
Subject: [PATCH 8/8] Update extract_node_from_dotfile.sh

---
 easybuild/scripts/extract_node_from_dotfile.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/easybuild/scripts/extract_node_from_dotfile.sh b/easybuild/scripts/extract_node_from_dotfile.sh
index a44454f556..2161448910 100755
--- a/easybuild/scripts/extract_node_from_dotfile.sh
+++ b/easybuild/scripts/extract_node_from_dotfile.sh
@@ -75,9 +75,9 @@ if [ "$#" -ne 3 ]; then
     exit 1
 fi
 
-local input_dotfile=$1
-local output_dotfile=$2
-local node_to_search_for=$3
+input_dotfile=$1
+output_dotfile=$2
+node_to_search_for=$3
 
 # Begin digraph in output file
 echo digraph graphname \{ > ${output_dotfile}