From 0f7bb982729afe3ddffbeec7b1fe6e4e6c6c29ec Mon Sep 17 00:00:00 2001
From: Filip Figiel <figiel.filip@gmail.com>
Date: Sun, 8 Oct 2023 01:42:46 +0200
Subject: [PATCH 01/12] fix: exclude retired packages when selecting new
 releases

---
 src/packages/syncing.gleam | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/packages/syncing.gleam b/src/packages/syncing.gleam
index 91e59d2..7d9e445 100644
--- a/src/packages/syncing.gleam
+++ b/src/packages/syncing.gleam
@@ -3,6 +3,7 @@ import birl/duration
 import gleam/dynamic as dyn
 import gleam/hackney
 import gleam/hexpm
+import gleam/option
 import gleam/http/request
 import gleam/int
 import gleam/json
@@ -218,7 +219,10 @@ fn lookup_gleam_releases(
   ))
   releases
   |> list.filter(fn(release) {
-    list.contains(release.meta.build_tools, "gleam")
+    // Select packages built with gleam, ignore retired releases
+    list.contains(release.meta.build_tools, "gleam") && option.is_none(
+      release.retirement,
+    )
   })
   |> Ok
 }

From 8a4f0c7750ac6aa1c3ed176d1d8b987ced205309 Mon Sep 17 00:00:00 2001
From: Filip Figiel <figiel.filip@gmail.com>
Date: Sun, 8 Oct 2023 01:43:42 +0200
Subject: [PATCH 02/12] feat: remove a package from index if there are no valid
 releases

---
 sql/delete_package.sql           |   1 +
 src/packages/generated/sql.gleam | 134 +++++++++++++++++--------------
 src/packages/index.gleam         |  12 ++-
 src/packages/syncing.gleam       |   2 +
 4 files changed, 87 insertions(+), 62 deletions(-)
 create mode 100644 sql/delete_package.sql

diff --git a/sql/delete_package.sql b/sql/delete_package.sql
new file mode 100644
index 0000000..bcffcf9
--- /dev/null
+++ b/sql/delete_package.sql
@@ -0,0 +1 @@
+delete from packages where name == $1;
diff --git a/src/packages/generated/sql.gleam b/src/packages/generated/sql.gleam
index ff51b90..58ddae4 100644
--- a/src/packages/generated/sql.gleam
+++ b/src/packages/generated/sql.gleam
@@ -9,21 +9,16 @@ import packages/error.{Error}
 pub type QueryResult(t) =
   Result(List(t), Error)
 
-pub fn get_most_recent_releases(
+pub fn get_total_package_count(
   db: sqlight.Connection,
   arguments: List(sqlight.Value),
   decoder: dynamic.Decoder(a),
 ) -> QueryResult(a) {
   let query =
     "select
-  version
+  count(1)
 from
-  releases
-where
-  package_id = $1
-order by
-  inserted_in_hex_at desc
-limit 5;
+  packages;
 "
   sqlight.query(query, db, arguments, decoder)
   |> result.map_error(error.DatabaseError)
@@ -65,19 +60,24 @@ returning
   |> result.map_error(error.DatabaseError)
 }
 
-pub fn upsert_most_recent_hex_timestamp(
+pub fn get_package(
   db: sqlight.Connection,
   arguments: List(sqlight.Value),
   decoder: dynamic.Decoder(a),
 ) -> QueryResult(a) {
   let query =
-    "insert into most_recent_hex_timestamp
-  (id, unix_timestamp)
-values
-  (1, $1)
-on conflict (id) do update
-set
-  unix_timestamp = $1;
+    "select
+  name
+, description
+, docs_url
+, links
+, inserted_in_hex_at
+, updated_in_hex_at
+from
+  packages
+where
+  id = $1
+limit 1;
 "
   sqlight.query(query, db, arguments, decoder)
   |> result.map_error(error.DatabaseError)
@@ -106,24 +106,19 @@ returning
   |> result.map_error(error.DatabaseError)
 }
 
-pub fn get_package(
+pub fn upsert_most_recent_hex_timestamp(
   db: sqlight.Connection,
   arguments: List(sqlight.Value),
   decoder: dynamic.Decoder(a),
 ) -> QueryResult(a) {
   let query =
-    "select
-  name
-, description
-, docs_url
-, links
-, inserted_in_hex_at
-, updated_in_hex_at
-from
-  packages
-where
-  id = $1
-limit 1;
+    "insert into most_recent_hex_timestamp
+  (id, unix_timestamp)
+values
+  (1, $1)
+on conflict (id) do update
+set
+  unix_timestamp = $1;
 "
   sqlight.query(query, db, arguments, decoder)
   |> result.map_error(error.DatabaseError)
@@ -144,55 +139,47 @@ limit 1
   |> result.map_error(error.DatabaseError)
 }
 
-pub fn get_total_package_count(
+pub fn json_dump(
   db: sqlight.Connection,
   arguments: List(sqlight.Value),
   decoder: dynamic.Decoder(a),
 ) -> QueryResult(a) {
   let query =
     "select
-  count(1)
-from
-  packages;
+  json_agg(row_to_json(packages))
+from packages;
 "
   sqlight.query(query, db, arguments, decoder)
   |> result.map_error(error.DatabaseError)
 }
 
-pub fn search_packages(
+pub fn delete_package(
+  db: sqlight.Connection,
+  arguments: List(sqlight.Value),
+  decoder: dynamic.Decoder(a),
+) -> QueryResult(a) {
+  let query =
+    "delete from packages where name == $1;
+"
+  sqlight.query(query, db, arguments, decoder)
+  |> result.map_error(error.DatabaseError)
+}
+
+pub fn get_most_recent_releases(
   db: sqlight.Connection,
   arguments: List(sqlight.Value),
   decoder: dynamic.Decoder(a),
 ) -> QueryResult(a) {
   let query =
     "select
-  id
-, name
-, description
-, docs_url
-, links
-, updated_in_hex_at
+  version
 from
-  packages
+  releases
 where
-  (
-    $1 = ''
-    or rowid in (
-      select rowid
-      from packages_fts
-      where packages_fts match $1
-    )
-  )
-  and not exists (
-    select 1
-    from hidden_packages
-    where hidden_packages.name = packages.name
-  )
-group by
-  packages.id
+  package_id = $1
 order by
-  packages.updated_in_hex_at desc
-limit 1000;
+  inserted_in_hex_at desc
+limit 5;
 "
   sqlight.query(query, db, arguments, decoder)
   |> result.map_error(error.DatabaseError)
@@ -221,15 +208,40 @@ limit 1;
   |> result.map_error(error.DatabaseError)
 }
 
-pub fn json_dump(
+pub fn search_packages(
   db: sqlight.Connection,
   arguments: List(sqlight.Value),
   decoder: dynamic.Decoder(a),
 ) -> QueryResult(a) {
   let query =
     "select
-  json_agg(row_to_json(packages))
-from packages;
+  id
+, name
+, description
+, docs_url
+, links
+, updated_in_hex_at
+from
+  packages
+where
+  (
+    $1 = ''
+    or rowid in (
+      select rowid
+      from packages_fts
+      where packages_fts match $1
+    )
+  )
+  and not exists (
+    select 1
+    from hidden_packages
+    where hidden_packages.name = packages.name
+  )
+group by
+  packages.id
+order by
+  packages.updated_in_hex_at desc
+limit 1000;
 "
   sqlight.query(query, db, arguments, decoder)
   |> result.map_error(error.DatabaseError)
diff --git a/src/packages/index.gleam b/src/packages/index.gleam
index 5d41fda..7a374dc 100644
--- a/src/packages/index.gleam
+++ b/src/packages/index.gleam
@@ -53,7 +53,7 @@ create table if not exists most_recent_hex_timestamp (
     -- we use a constraint to enforce that the id is always the value `1` so
     -- now this table can only hold one row.
     check (id == 1),
-  
+
   unix_timestamp integer not null
 ) strict;
 
@@ -237,6 +237,16 @@ pub fn upsert_package(
   Ok(id)
 }
 
+pub fn delete_package(db: Connection, name: String) -> Result(Nil, Error) {
+  let parameters = [sqlight.text(name)]
+  use _returned <- result.then(sql.delete_package(
+    db.inner,
+    parameters,
+    dyn.dynamic,
+  ))
+  Ok(Nil)
+}
+
 pub type Package {
   Package(
     name: String,
diff --git a/src/packages/syncing.gleam b/src/packages/syncing.gleam
index 7d9e445..3c4c99d 100644
--- a/src/packages/syncing.gleam
+++ b/src/packages/syncing.gleam
@@ -180,6 +180,7 @@ fn sync_package(state: State, package: hexpm.Package) -> Result(State, Error) {
 
   case releases {
     [] -> {
+      use _ <- try(index.delete_package(state.db, package.name))
       let state = log_if_needed(state, package.updated_at)
       Ok(state)
     }
@@ -200,6 +201,7 @@ fn sync_single_package(
 
   case releases {
     [] -> {
+      use _ <- try(index.delete_package(db, package.name))
       Ok(Nil)
     }
     _ -> {

From 1423bf97a600b762c8e8d3606b35d805c960865f Mon Sep 17 00:00:00 2001
From: Filip Figiel <figiel.filip@gmail.com>
Date: Fri, 26 Jan 2024 19:39:25 +0100
Subject: [PATCH 03/12] chore: add comments

---
 src/packages/syncing.gleam | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/packages/syncing.gleam b/src/packages/syncing.gleam
index 3c4c99d..d3969ec 100644
--- a/src/packages/syncing.gleam
+++ b/src/packages/syncing.gleam
@@ -180,6 +180,7 @@ fn sync_package(state: State, package: hexpm.Package) -> Result(State, Error) {
 
   case releases {
     [] -> {
+      // Delete the package in case it was present in the index but all its releases have been retired.
       use _ <- try(index.delete_package(state.db, package.name))
       let state = log_if_needed(state, package.updated_at)
       Ok(state)
@@ -201,6 +202,7 @@ fn sync_single_package(
 
   case releases {
     [] -> {
+      // Delete the package in case it was present in the index but all its releases have been retired.
       use _ <- try(index.delete_package(db, package.name))
       Ok(Nil)
     }

From 07e346c50093c7139b130b39d95454b6e04294e0 Mon Sep 17 00:00:00 2001
From: Filip Figiel <figiel.filip@gmail.com>
Date: Fri, 26 Jan 2024 19:43:01 +0100
Subject: [PATCH 04/12] refactor: run gleam format

---
 src/packages/syncing.gleam | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/packages/syncing.gleam b/src/packages/syncing.gleam
index 4d67b6c..750b94c 100644
--- a/src/packages/syncing.gleam
+++ b/src/packages/syncing.gleam
@@ -225,9 +225,8 @@ fn lookup_gleam_releases(
   releases
   |> list.filter(fn(release) {
     // Select packages built with gleam, ignore retired releases
-    list.contains(release.meta.build_tools, "gleam") && option.is_none(
-      release.retirement,
-    )
+    list.contains(release.meta.build_tools, "gleam")
+    && option.is_none(release.retirement)
   })
   |> Ok
 }

From 24454af3043503d1f9d6f3f42983215d4d9100ce Mon Sep 17 00:00:00 2001
From: Filip Figiel <figiel.filip@gmail.com>
Date: Mon, 29 Jan 2024 20:02:10 +0100
Subject: [PATCH 05/12] fix: upgrade codegen to use new syntax

---
 test/codegen.gleam | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/codegen.gleam b/test/codegen.gleam
index 804f50d..d9763b6 100644
--- a/test/codegen.gleam
+++ b/test/codegen.gleam
@@ -17,7 +17,7 @@ fn generate_sql_queries_module() -> Nil {
 
   let imports = [
     "import sqlight", "import gleam/result", "import gleam/dynamic",
-    "import packages/error.{Error}",
+    "import packages/error.{type Error}",
   ]
   let module =
     string.join(

From 0abd0d366c2e636d3fd278e7db52e646ae0d94fc Mon Sep 17 00:00:00 2001
From: Filip Figiel <figiel.filip@gmail.com>
Date: Mon, 29 Jan 2024 20:05:56 +0100
Subject: [PATCH 06/12] fix: do not delete retired packages

---
 sql/delete_package.sql           |  1 -
 src/packages/generated/sql.gleam | 14 +-------------
 src/packages/index.gleam         | 10 ----------
 src/packages/syncing.gleam       |  7 -------
 test/codegen.gleam               |  2 +-
 5 files changed, 2 insertions(+), 32 deletions(-)
 delete mode 100644 sql/delete_package.sql

diff --git a/sql/delete_package.sql b/sql/delete_package.sql
deleted file mode 100644
index bcffcf9..0000000
--- a/sql/delete_package.sql
+++ /dev/null
@@ -1 +0,0 @@
-delete from packages where name == $1;
diff --git a/src/packages/generated/sql.gleam b/src/packages/generated/sql.gleam
index 2cb2e13..6a2de0f 100644
--- a/src/packages/generated/sql.gleam
+++ b/src/packages/generated/sql.gleam
@@ -1,4 +1,4 @@
-// THIS FILE IS GENERATED. DO NOT EDIT. 
+// THIS FILE IS GENERATED. DO NOT EDIT.
 // Regenerate with `gleam run -m codegen`
 
 import sqlight
@@ -153,18 +153,6 @@ from packages;
   |> result.map_error(error.DatabaseError)
 }
 
-pub fn delete_package(
-  db: sqlight.Connection,
-  arguments: List(sqlight.Value),
-  decoder: dynamic.Decoder(a),
-) -> QueryResult(a) {
-  let query =
-    "delete from packages where name == $1;
-"
-  sqlight.query(query, db, arguments, decoder)
-  |> result.map_error(error.DatabaseError)
-}
-
 pub fn get_most_recent_releases(
   db: sqlight.Connection,
   arguments: List(sqlight.Value),
diff --git a/src/packages/index.gleam b/src/packages/index.gleam
index 1f651df..fa4c68c 100644
--- a/src/packages/index.gleam
+++ b/src/packages/index.gleam
@@ -238,16 +238,6 @@ pub fn upsert_package(
   Ok(id)
 }
 
-pub fn delete_package(db: Connection, name: String) -> Result(Nil, Error) {
-  let parameters = [sqlight.text(name)]
-  use _returned <- result.then(sql.delete_package(
-    db.inner,
-    parameters,
-    dyn.dynamic,
-  ))
-  Ok(Nil)
-}
-
 pub type Package {
   Package(
     name: String,
diff --git a/src/packages/syncing.gleam b/src/packages/syncing.gleam
index 750b94c..c471f63 100644
--- a/src/packages/syncing.gleam
+++ b/src/packages/syncing.gleam
@@ -3,7 +3,6 @@ import birl/duration
 import gleam/dynamic as dyn
 import gleam/hackney
 import gleam/hexpm
-import gleam/option
 import gleam/http/request
 import gleam/int
 import gleam/json
@@ -182,8 +181,6 @@ fn sync_package(state: State, package: hexpm.Package) -> Result(State, Error) {
 
   case releases {
     [] -> {
-      // Delete the package in case it was present in the index but all its releases have been retired.
-      use _ <- try(index.delete_package(state.db, package.name))
       let state = log_if_needed(state, package.updated_at)
       Ok(state)
     }
@@ -204,8 +201,6 @@ fn sync_single_package(
 
   case releases {
     [] -> {
-      // Delete the package in case it was present in the index but all its releases have been retired.
-      use _ <- try(index.delete_package(db, package.name))
       Ok(Nil)
     }
     _ -> {
@@ -224,9 +219,7 @@ fn lookup_gleam_releases(
   )
   releases
   |> list.filter(fn(release) {
-    // Select packages built with gleam, ignore retired releases
     list.contains(release.meta.build_tools, "gleam")
-    && option.is_none(release.retirement)
   })
   |> Ok
 }
diff --git a/test/codegen.gleam b/test/codegen.gleam
index d9763b6..67e8e3d 100644
--- a/test/codegen.gleam
+++ b/test/codegen.gleam
@@ -7,7 +7,7 @@ pub fn main() {
   let Nil = generate_sql_queries_module()
 }
 
-const module_header = "// THIS FILE IS GENERATED. DO NOT EDIT. 
+const module_header = "// THIS FILE IS GENERATED. DO NOT EDIT.
 // Regenerate with `gleam run -m codegen`"
 
 fn generate_sql_queries_module() -> Nil {

From 5e37b3100e701547b757f374f78ae4aeea52813e Mon Sep 17 00:00:00 2001
From: Filip Figiel <figiel.filip@gmail.com>
Date: Mon, 29 Jan 2024 20:10:41 +0100
Subject: [PATCH 07/12] fix: use a subquery to exclude retired packages

---
 sql/get_package.sql              | 13 +++++++++
 sql/get_total_package_count.sql  | 16 ++++++++++-
 sql/search_packages.sql          | 13 +++++++++
 src/packages/generated/sql.gleam | 48 +++++++++++++++++++++++++++++---
 test/packages/index_test.gleam   | 16 ++---------
 5 files changed, 87 insertions(+), 19 deletions(-)

diff --git a/sql/get_package.sql b/sql/get_package.sql
index c97f5c8..cdb9e56 100644
--- a/sql/get_package.sql
+++ b/sql/get_package.sql
@@ -1,3 +1,15 @@
+with
+  retired_package_ids as (
+    select r.package_id from releases r where exists (
+      select
+        1
+      from
+        releases
+      where
+        r.package_id = package_id
+        and retirement_message is not null
+    )
+  )
 select
   name
 , description
@@ -9,4 +21,5 @@ from
   packages
 where
   id = $1
+  and id not in retired_package_ids
 limit 1;
diff --git a/sql/get_total_package_count.sql b/sql/get_total_package_count.sql
index ca62c2a..c33634a 100644
--- a/sql/get_total_package_count.sql
+++ b/sql/get_total_package_count.sql
@@ -1,4 +1,18 @@
+with
+  retired_package_ids as (
+    select r.package_id from releases r where exists (
+      select
+        1
+      from
+        releases
+      where
+        r.package_id = package_id
+        and retirement_message is not null
+    )
+  )
 select
   count(1)
 from
-  packages;
+  packages
+where
+  id not in retired_package_ids;
diff --git a/sql/search_packages.sql b/sql/search_packages.sql
index e47eaac..dd9db73 100644
--- a/sql/search_packages.sql
+++ b/sql/search_packages.sql
@@ -1,3 +1,15 @@
+with
+  retired_package_ids as (
+    select r.package_id from releases r where exists (
+      select
+        1
+      from
+        releases
+      where
+        r.package_id = package_id
+        and retirement_message is not null
+    )
+  )
 select
   id
 , name
@@ -21,6 +33,7 @@ where
     from hidden_packages
     where hidden_packages.name = packages.name
   )
+  and id not in retired_package_ids
 group by
   packages.id
 order by
diff --git a/src/packages/generated/sql.gleam b/src/packages/generated/sql.gleam
index 6a2de0f..afface0 100644
--- a/src/packages/generated/sql.gleam
+++ b/src/packages/generated/sql.gleam
@@ -15,10 +15,24 @@ pub fn get_total_package_count(
   decoder: dynamic.Decoder(a),
 ) -> QueryResult(a) {
   let query =
-    "select
+    "with
+  retired_package_ids as (
+    select r.package_id from releases r where exists (
+      select
+        1
+      from
+        releases
+      where
+        r.package_id = package_id
+        and retirement_message is not null
+    )
+  )
+select
   count(1)
 from
-  packages;
+  packages
+where
+  id not in retired_package_ids;
 "
   sqlight.query(query, db, arguments, decoder)
   |> result.map_error(error.DatabaseError)
@@ -66,7 +80,19 @@ pub fn get_package(
   decoder: dynamic.Decoder(a),
 ) -> QueryResult(a) {
   let query =
-    "select
+    "with
+  retired_package_ids as (
+    select r.package_id from releases r where exists (
+      select
+        1
+      from
+        releases
+      where
+        r.package_id = package_id
+        and retirement_message is not null
+    )
+  )
+select
   name
 , description
 , docs_url
@@ -77,6 +103,7 @@ from
   packages
 where
   id = $1
+  and id not in retired_package_ids
 limit 1;
 "
   sqlight.query(query, db, arguments, decoder)
@@ -202,7 +229,19 @@ pub fn search_packages(
   decoder: dynamic.Decoder(a),
 ) -> QueryResult(a) {
   let query =
-    "select
+    "with
+  retired_package_ids as (
+    select r.package_id from releases r where exists (
+      select
+        1
+      from
+        releases
+      where
+        r.package_id = package_id
+        and retirement_message is not null
+    )
+  )
+select
   id
 , name
 , description
@@ -225,6 +264,7 @@ where
     from hidden_packages
     where hidden_packages.name = packages.name
   )
+  and id not in retired_package_ids
 group by
   packages.id
 order by
diff --git a/test/packages/index_test.gleam b/test/packages/index_test.gleam
index 72e32ef..fd065e6 100644
--- a/test/packages/index_test.gleam
+++ b/test/packages/index_test.gleam
@@ -206,20 +206,8 @@ pub fn search_packages_empty_test() {
 
   let assert Ok(packages) = index.search_packages(db, "library")
   packages
-  |> should.equal([
-    index.PackageSummary(
-      id: package_id,
-      name: "gleam_stdlib",
-      description: "Standard library for Gleam",
-      docs_url: Some("https://hexdocs.pm/gleam_stdlib/"),
-      links: dict.from_list([
-        #("Website", "https://gleam.run/"),
-        #("Repository", "https://github.com/gleam-lang/stdlib"),
-      ]),
-      latest_versions: ["0.0.4", "0.0.3"],
-      updated_in_hex_at: birl.from_unix(2000),
-    ),
-  ])
+  |> should.equal([])
+  // No results because all releases of gleam_stdlib are retired
   // TODO: include latest versions
 }
 

From 9e77c15b9566edc13adc628bf2ee8c9613e5b251 Mon Sep 17 00:00:00 2001
From: Filip Figiel <figiel.filip@gmail.com>
Date: Mon, 29 Jan 2024 21:15:41 +0100
Subject: [PATCH 08/12] fix: solve issues with wrong packages being hidden, add
 tests

---
 sql/get_package.sql              | 15 ++++--
 sql/get_total_package_count.sql  | 15 ++++--
 sql/search_packages.sql          | 15 ++++--
 src/packages/generated/sql.gleam | 45 +++++++++++------
 test/packages/index_test.gleam   | 86 ++++++++++++++++++++++++++++++++
 5 files changed, 146 insertions(+), 30 deletions(-)

diff --git a/sql/get_package.sql b/sql/get_package.sql
index cdb9e56..73383db 100644
--- a/sql/get_package.sql
+++ b/sql/get_package.sql
@@ -1,14 +1,19 @@
 with
   retired_package_ids as (
-    select r.package_id from releases r where exists (
+    -- A package is retired if its latest release is retired
+    select package_id from (
       select
-        1
+        package_id
+        , retirement_reason
       from
         releases
-      where
-        r.package_id = package_id
-        and retirement_message is not null
+      group by
+        package_id
+      having
+        max(inserted_in_hex_at)
     )
+    where
+      retirement_reason is not null
   )
 select
   name
diff --git a/sql/get_total_package_count.sql b/sql/get_total_package_count.sql
index c33634a..4170edd 100644
--- a/sql/get_total_package_count.sql
+++ b/sql/get_total_package_count.sql
@@ -1,14 +1,19 @@
 with
   retired_package_ids as (
-    select r.package_id from releases r where exists (
+    -- A package is retired if its latest release is retired
+    select package_id from (
       select
-        1
+        package_id
+        , retirement_reason
       from
         releases
-      where
-        r.package_id = package_id
-        and retirement_message is not null
+      group by
+        package_id
+      having
+        max(inserted_in_hex_at)
     )
+    where
+      retirement_reason is not null
   )
 select
   count(1)
diff --git a/sql/search_packages.sql b/sql/search_packages.sql
index dd9db73..ca6ea80 100644
--- a/sql/search_packages.sql
+++ b/sql/search_packages.sql
@@ -1,14 +1,19 @@
 with
   retired_package_ids as (
-    select r.package_id from releases r where exists (
+    -- A package is retired if its latest release is retired
+    select package_id from (
       select
-        1
+        package_id
+        , retirement_reason
       from
         releases
-      where
-        r.package_id = package_id
-        and retirement_message is not null
+      group by
+        package_id
+      having
+        max(inserted_in_hex_at)
     )
+    where
+      retirement_reason is not null
   )
 select
   id
diff --git a/src/packages/generated/sql.gleam b/src/packages/generated/sql.gleam
index afface0..2fad22a 100644
--- a/src/packages/generated/sql.gleam
+++ b/src/packages/generated/sql.gleam
@@ -17,15 +17,20 @@ pub fn get_total_package_count(
   let query =
     "with
   retired_package_ids as (
-    select r.package_id from releases r where exists (
+    -- A package is retired if its latest release is retired
+    select package_id from (
       select
-        1
+        package_id
+        , retirement_reason
       from
         releases
-      where
-        r.package_id = package_id
-        and retirement_message is not null
+      group by
+        package_id
+      having
+        max(inserted_in_hex_at)
     )
+    where
+      retirement_reason is not null
   )
 select
   count(1)
@@ -82,15 +87,20 @@ pub fn get_package(
   let query =
     "with
   retired_package_ids as (
-    select r.package_id from releases r where exists (
+    -- A package is retired if its latest release is retired
+    select package_id from (
       select
-        1
+        package_id
+        , retirement_reason
       from
         releases
-      where
-        r.package_id = package_id
-        and retirement_message is not null
+      group by
+        package_id
+      having
+        max(inserted_in_hex_at)
     )
+    where
+      retirement_reason is not null
   )
 select
   name
@@ -231,15 +241,20 @@ pub fn search_packages(
   let query =
     "with
   retired_package_ids as (
-    select r.package_id from releases r where exists (
+    -- A package is retired if its latest release is retired
+    select package_id from (
       select
-        1
+        package_id
+        , retirement_reason
       from
         releases
-      where
-        r.package_id = package_id
-        and retirement_message is not null
+      group by
+        package_id
+      having
+        max(inserted_in_hex_at)
     )
+    where
+      retirement_reason is not null
   )
 select
   id
diff --git a/test/packages/index_test.gleam b/test/packages/index_test.gleam
index fd065e6..a8d9b3e 100644
--- a/test/packages/index_test.gleam
+++ b/test/packages/index_test.gleam
@@ -1,5 +1,6 @@
 import packages/index.{Package, Release}
 import gleam/hexpm
+import gleam/list
 import gleam/option.{None, Some}
 import gleam/dict
 import gleeunit/should
@@ -211,6 +212,91 @@ pub fn search_packages_empty_test() {
   // TODO: include latest versions
 }
 
+pub fn search_packages_hide_retired_test() {
+  use db <- tests.with_database
+
+  let assert Ok(package_id) =
+    index.upsert_package(
+      db,
+      hexpm.Package(
+        downloads: dict.from_list([#("all", 5), #("recent", 2)]),
+        docs_html_url: Some("https://hexdocs.pm/gleam_stdlib/"),
+        html_url: Some("https://hex.pm/packages/gleam_stdlib"),
+        meta: hexpm.PackageMeta(
+          description: Some("Standard library for Gleam"),
+          licenses: ["Apache-2.0"],
+          links: dict.from_list([
+            #("Website", "https://gleam.run/"),
+            #("Repository", "https://github.com/gleam-lang/stdlib"),
+          ]),
+        ),
+        name: "gleam_stdlib",
+        owners: None,
+        releases: [],
+        inserted_at: birl.from_unix(100),
+        updated_at: birl.from_unix(2000),
+      ),
+    )
+
+  let base_release =
+    hexpm.Release(
+      version: "0.0.3",
+      checksum: "a895b55c4c3749eb32328f02b15bbd3acc205dd874fabd135d7be5d12eda59a8",
+      url: "https://hex.pm/api/packages/shimmer/releases/0.0.3",
+      downloads: 0,
+      meta: hexpm.ReleaseMeta(app: Some("shimmer"), build_tools: ["gleam"]),
+      publisher: Some(hexpm.PackageOwner(
+        username: "harryet",
+        email: None,
+        url: "https://hex.pm/api/users/harryet",
+      )),
+      retirement: None,
+      updated_at: birl.from_unix(1000),
+      inserted_at: birl.from_unix(2000),
+    )
+
+  // A package's first release is published, it should be visible
+  let assert Ok(_) = index.upsert_release(db, package_id, base_release)
+  let assert Ok(packages) = index.search_packages(db, "gleam_stdlib")
+  list.length(packages)
+  |> should.equal(1)
+
+  // The release is retired, the package should now be hidden
+  let assert Ok(_) =
+    index.upsert_release(
+      db,
+      package_id,
+      hexpm.Release(
+        ..base_release,
+        retirement: Some(hexpm.ReleaseRetirement(
+          reason: hexpm.Security,
+          message: Some("Retired due to security concerns"),
+        )),
+        updated_at: birl.from_unix(1001),
+        inserted_at: birl.from_unix(2001),
+      ),
+    )
+  let assert Ok(packages) = index.search_packages(db, "gleam_stdlib")
+  list.length(packages)
+  |> should.equal(0)
+
+  // A new release is published, the package should be visible again
+  let assert Ok(_) =
+    index.upsert_release(
+      db,
+      package_id,
+      hexpm.Release(
+        ..base_release,
+        version: "0.0.4",
+        updated_at: birl.from_unix(1002),
+        inserted_at: birl.from_unix(2002),
+      ),
+    )
+  let assert Ok(packages) = index.search_packages(db, "gleam_stdlib")
+  list.length(packages)
+  |> should.equal(1)
+}
+
 pub fn search_packages_query_escaping_test() {
   use db <- tests.with_database
   let assert Ok(package_id) =

From 1d99a4377c0573c745b19ebe91b7aa43adaec740 Mon Sep 17 00:00:00 2001
From: Filip Figiel <figiel.filip@gmail.com>
Date: Tue, 30 Jan 2024 23:43:35 +0100
Subject: [PATCH 09/12] refactor: use a sql view to reduce repetitive filters

---
 sql/get_package.sql              | 18 ----------
 sql/get_total_package_count.sql  | 17 ----------
 sql/search_packages.sql          | 17 ----------
 src/packages/generated/sql.gleam | 58 ++------------------------------
 src/packages/index.gleam         | 16 +++++++++
 5 files changed, 19 insertions(+), 107 deletions(-)

diff --git a/sql/get_package.sql b/sql/get_package.sql
index 73383db..c97f5c8 100644
--- a/sql/get_package.sql
+++ b/sql/get_package.sql
@@ -1,20 +1,3 @@
-with
-  retired_package_ids as (
-    -- A package is retired if its latest release is retired
-    select package_id from (
-      select
-        package_id
-        , retirement_reason
-      from
-        releases
-      group by
-        package_id
-      having
-        max(inserted_in_hex_at)
-    )
-    where
-      retirement_reason is not null
-  )
 select
   name
 , description
@@ -26,5 +9,4 @@ from
   packages
 where
   id = $1
-  and id not in retired_package_ids
 limit 1;
diff --git a/sql/get_total_package_count.sql b/sql/get_total_package_count.sql
index 4170edd..0f98cb0 100644
--- a/sql/get_total_package_count.sql
+++ b/sql/get_total_package_count.sql
@@ -1,20 +1,3 @@
-with
-  retired_package_ids as (
-    -- A package is retired if its latest release is retired
-    select package_id from (
-      select
-        package_id
-        , retirement_reason
-      from
-        releases
-      group by
-        package_id
-      having
-        max(inserted_in_hex_at)
-    )
-    where
-      retirement_reason is not null
-  )
 select
   count(1)
 from
diff --git a/sql/search_packages.sql b/sql/search_packages.sql
index ca6ea80..31420ec 100644
--- a/sql/search_packages.sql
+++ b/sql/search_packages.sql
@@ -1,20 +1,3 @@
-with
-  retired_package_ids as (
-    -- A package is retired if its latest release is retired
-    select package_id from (
-      select
-        package_id
-        , retirement_reason
-      from
-        releases
-      group by
-        package_id
-      having
-        max(inserted_in_hex_at)
-    )
-    where
-      retirement_reason is not null
-  )
 select
   id
 , name
diff --git a/src/packages/generated/sql.gleam b/src/packages/generated/sql.gleam
index 2fad22a..99cf96b 100644
--- a/src/packages/generated/sql.gleam
+++ b/src/packages/generated/sql.gleam
@@ -15,24 +15,7 @@ pub fn get_total_package_count(
   decoder: dynamic.Decoder(a),
 ) -> QueryResult(a) {
   let query =
-    "with
-  retired_package_ids as (
-    -- A package is retired if its latest release is retired
-    select package_id from (
-      select
-        package_id
-        , retirement_reason
-      from
-        releases
-      group by
-        package_id
-      having
-        max(inserted_in_hex_at)
-    )
-    where
-      retirement_reason is not null
-  )
-select
+    "select
   count(1)
 from
   packages
@@ -85,24 +68,7 @@ pub fn get_package(
   decoder: dynamic.Decoder(a),
 ) -> QueryResult(a) {
   let query =
-    "with
-  retired_package_ids as (
-    -- A package is retired if its latest release is retired
-    select package_id from (
-      select
-        package_id
-        , retirement_reason
-      from
-        releases
-      group by
-        package_id
-      having
-        max(inserted_in_hex_at)
-    )
-    where
-      retirement_reason is not null
-  )
-select
+    "select
   name
 , description
 , docs_url
@@ -113,7 +79,6 @@ from
   packages
 where
   id = $1
-  and id not in retired_package_ids
 limit 1;
 "
   sqlight.query(query, db, arguments, decoder)
@@ -239,24 +204,7 @@ pub fn search_packages(
   decoder: dynamic.Decoder(a),
 ) -> QueryResult(a) {
   let query =
-    "with
-  retired_package_ids as (
-    -- A package is retired if its latest release is retired
-    select package_id from (
-      select
-        package_id
-        , retirement_reason
-      from
-        releases
-      group by
-        package_id
-      having
-        max(inserted_in_hex_at)
-    )
-    where
-      retirement_reason is not null
-  )
-select
+    "select
   id
 , name
 , description
diff --git a/src/packages/index.gleam b/src/packages/index.gleam
index fa4c68c..52f2b15 100644
--- a/src/packages/index.gleam
+++ b/src/packages/index.gleam
@@ -141,6 +141,22 @@ create table if not exists hidden_packages (
   name text primary key
 ) strict;
 
+create view if not exists retired_package_ids as
+  -- A package is retired if its latest release is retired
+  select package_id from (
+    select
+      package_id
+      , retirement_reason
+    from
+      releases
+    group by
+      package_id
+    having
+      max(inserted_in_hex_at)
+  )
+  where
+    retirement_reason is not null;
+
 -- These packages are placeholders or otherwise not useful.
 insert into hidden_packages values
   -- Test packages.

From b3736a8a2f880bec9637e26e16e0343c39d6857c Mon Sep 17 00:00:00 2001
From: Filip Figiel <figiel.filip@gmail.com>
Date: Tue, 13 Feb 2024 00:36:24 +0100
Subject: [PATCH 10/12] fix(perf): use a `non_retired_packages` view to select
 packages

---
 sql/get_total_package_count.sql  |  4 +---
 sql/search_packages.sql          | 11 +++++------
 src/packages/generated/sql.gleam | 15 ++++++---------
 src/packages/index.gleam         | 30 +++++++++++++++++-------------
 test/packages/index_test.gleam   |  1 +
 5 files changed, 30 insertions(+), 31 deletions(-)

diff --git a/sql/get_total_package_count.sql b/sql/get_total_package_count.sql
index 0f98cb0..ac615a7 100644
--- a/sql/get_total_package_count.sql
+++ b/sql/get_total_package_count.sql
@@ -1,6 +1,4 @@
 select
   count(1)
 from
-  packages
-where
-  id not in retired_package_ids;
+  non_retired_packages;
diff --git a/sql/search_packages.sql b/sql/search_packages.sql
index 31420ec..c399a87 100644
--- a/sql/search_packages.sql
+++ b/sql/search_packages.sql
@@ -6,11 +6,11 @@ select
 , links
 , updated_in_hex_at
 from
-  packages
+  non_retired_packages p
 where
   (
     $1 = ''
-    or rowid in (
+    or id in (
       select rowid
       from packages_fts
       where packages_fts match $1
@@ -19,11 +19,10 @@ where
   and not exists (
     select 1
     from hidden_packages
-    where hidden_packages.name = packages.name
+    where hidden_packages.name = p.name
   )
-  and id not in retired_package_ids
 group by
-  packages.id
+  p.id
 order by
-  packages.updated_in_hex_at desc
+  p.updated_in_hex_at desc
 limit 1000;
diff --git a/src/packages/generated/sql.gleam b/src/packages/generated/sql.gleam
index 99cf96b..0820dee 100644
--- a/src/packages/generated/sql.gleam
+++ b/src/packages/generated/sql.gleam
@@ -18,9 +18,7 @@ pub fn get_total_package_count(
     "select
   count(1)
 from
-  packages
-where
-  id not in retired_package_ids;
+  non_retired_packages;
 "
   sqlight.query(query, db, arguments, decoder)
   |> result.map_error(error.DatabaseError)
@@ -212,11 +210,11 @@ pub fn search_packages(
 , links
 , updated_in_hex_at
 from
-  packages
+  non_retired_packages p
 where
   (
     $1 = ''
-    or rowid in (
+    or id in (
       select rowid
       from packages_fts
       where packages_fts match $1
@@ -225,13 +223,12 @@ where
   and not exists (
     select 1
     from hidden_packages
-    where hidden_packages.name = packages.name
+    where hidden_packages.name = p.name
   )
-  and id not in retired_package_ids
 group by
-  packages.id
+  p.id
 order by
-  packages.updated_in_hex_at desc
+  p.updated_in_hex_at desc
 limit 1000;
 "
   sqlight.query(query, db, arguments, decoder)
diff --git a/src/packages/index.gleam b/src/packages/index.gleam
index 52f2b15..2939835 100644
--- a/src/packages/index.gleam
+++ b/src/packages/index.gleam
@@ -141,21 +141,25 @@ create table if not exists hidden_packages (
   name text primary key
 ) strict;
 
-create view if not exists retired_package_ids as
+create view if not exists non_retired_packages as
   -- A package is retired if its latest release is retired
-  select package_id from (
-    select
-      package_id
-      , retirement_reason
-    from
-      releases
-    group by
-      package_id
-    having
-      max(inserted_in_hex_at)
-  )
+  select
+    p.id
+  , p.name
+  , p.description
+  , p.docs_url
+  , p.links
+  , p.updated_in_hex_at
+  , r1.retirement_reason
+  from
+    packages p
+  inner join releases r1
+      on p.id = r1.package_id
+  left outer join releases r2
+      on (p.id = r2.package_id and r1.id < r2.id)
   where
-    retirement_reason is not null;
+      r2.id is null
+      and r1.retirement_reason is null;
 
 -- These packages are placeholders or otherwise not useful.
 insert into hidden_packages values
diff --git a/test/packages/index_test.gleam b/test/packages/index_test.gleam
index a8d9b3e..f965dfa 100644
--- a/test/packages/index_test.gleam
+++ b/test/packages/index_test.gleam
@@ -215,6 +215,7 @@ pub fn search_packages_empty_test() {
 pub fn search_packages_hide_retired_test() {
   use db <- tests.with_database
 
+  // Prepare test data
   let assert Ok(package_id) =
     index.upsert_package(
       db,

From 6959d21caf95e0c5a708bdf5a4e43187ce1a1191 Mon Sep 17 00:00:00 2001
From: Filip Figiel <figiel.filip@gmail.com>
Date: Tue, 13 Feb 2024 00:38:18 +0100
Subject: [PATCH 11/12] refactor: remove unused column from view

---
 src/packages/index.gleam | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/packages/index.gleam b/src/packages/index.gleam
index 2939835..1bf9603 100644
--- a/src/packages/index.gleam
+++ b/src/packages/index.gleam
@@ -150,7 +150,6 @@ create view if not exists non_retired_packages as
   , p.docs_url
   , p.links
   , p.updated_in_hex_at
-  , r1.retirement_reason
   from
     packages p
   inner join releases r1

From 4f194623f1cda34509a6ffb761ed9c7f01abca05 Mon Sep 17 00:00:00 2001
From: Filip Figiel <figiel.filip@gmail.com>
Date: Thu, 15 Feb 2024 21:40:42 +0100
Subject: [PATCH 12/12] fix: use correct logic for selecting non-retired
 packages

---
 src/packages/index.gleam | 25 ++++++++-----------------
 1 file changed, 8 insertions(+), 17 deletions(-)

diff --git a/src/packages/index.gleam b/src/packages/index.gleam
index 1bf9603..461c911 100644
--- a/src/packages/index.gleam
+++ b/src/packages/index.gleam
@@ -142,23 +142,14 @@ create table if not exists hidden_packages (
 ) strict;
 
 create view if not exists non_retired_packages as
-  -- A package is retired if its latest release is retired
-  select
-    p.id
-  , p.name
-  , p.description
-  , p.docs_url
-  , p.links
-  , p.updated_in_hex_at
-  from
-    packages p
-  inner join releases r1
-      on p.id = r1.package_id
-  left outer join releases r2
-      on (p.id = r2.package_id and r1.id < r2.id)
-  where
-      r2.id is null
-      and r1.retirement_reason is null;
+  -- A package is retired if all its releases are retired
+  select p.*
+  from packages p
+  where p.id in (
+    select distinct r.package_id
+    from releases r
+    where r.id is null or r.retirement_reason is null
+  );
 
 -- These packages are placeholders or otherwise not useful.
 insert into hidden_packages values