fix: reqwest https (#79)

Author: definfo

Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 authors = ["Alex Chi <iskyzh@gmail.com>"]
-edition = "2018"
+edition = "2024"
 name = "mirror-intel"
 license = "Apache-2.0"
 version = "0.1.0"
@@ -20,9 +20,9 @@ lazy_static = "1.4"
 percent-encoding = "2.1"
 prometheus = "0.14"
 regex = "1"
-reqwest = { version = "0.13", default-features = false, features = ["stream"] }
+reqwest = { version = "0.13", default-features = false, features = ["stream", "rustls"] }
 rstest = "0.26"
-serde = "1.0"
+serde = { version = "1.0", features = ["derive"] }
 thiserror = "2.0"
 tokio = { version = "1", features = ["full"] }
 tracing = "0.1"

Cargo.toml

@@ -3,15 +3,15 @@
 use std::borrow::Cow;
 use std::collections::HashSet;
 use std::path::{Path, PathBuf};
-use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::Arc;
+use std::sync::atomic::{AtomicUsize, Ordering};
 
 use bytes::Bytes;
 use futures_util::StreamExt;
 use reqwest::{Client, Response};
 use tokio::fs::{self, OpenOptions};
 use tokio::io::{AsyncWriteExt, BufWriter};
-use tokio::sync::mpsc::{unbounded_channel, Receiver, UnboundedSender};
+use tokio::sync::mpsc::{Receiver, UnboundedSender, unbounded_channel};
 use tokio::sync::{Mutex, OwnedSemaphorePermit, Semaphore};
 use tracing::{info, instrument, warn};
 use url::Url;
@@ -56,10 +56,10 @@ fn next_buffer_path(config: &Config) -> PathBuf {
 
 /// Remove a temporary file, ignoring not-found errors.
 async fn remove_buffer_file(path: &Path) {
-    if let Err(err) = fs::remove_file(path).await {
-        if err.kind() != std::io::ErrorKind::NotFound {
-            warn!(error=?err, path=%path.display(), "failed to remove cache file");
-        }
+    if let Err(err) = fs::remove_file(path).await
+        && err.kind() != std::io::ErrorKind::NotFound
+    {
+        warn!(error=?err, path=%path.display(), "failed to remove cache file");
     }
 }
 

src/artifacts.rs

@@ -4,7 +4,7 @@ use std::borrow::Cow;
 use std::time::Duration;
 
 use actix_web::http::header::ContentType;
-use actix_web::{web, HttpResponse};
+use actix_web::{HttpResponse, web};
 
 use crate::common::{Config, IntelMission, IntelResponse, Redirect};
 use crate::intel_path::IntelPath;

src/browse.rs

@@ -1,13 +1,13 @@
 //! Common types.
 
 use actix_web::body::EitherBody;
-use actix_web::http::{header, StatusCode};
+use actix_web::http::{StatusCode, header};
 use actix_web::{HttpRequest, HttpResponse, Responder};
 use aws_sdk_s3::Client as S3Client;
-use figment::providers::{Format, Serialized, Toml};
 use figment::Figment;
+use figment::providers::{Format, Serialized, Toml};
 use percent_encoding::percent_decode;
-use prometheus::{proto, IntCounter as Counter, IntGauge as Gauge, Opts, Registry};
+use prometheus::{IntCounter as Counter, IntGauge as Gauge, Opts, Registry, proto};
 use reqwest::Client;
 use serde::Deserialize;
 use std::path::PathBuf;
@@ -373,10 +373,10 @@ mod tests {
     use figment::Jail;
     use serial_test::serial;
 
+    use crate::Config;
     use crate::common::{
-        collect_config, EndpointOverride, Endpoints, GithubReleaseConfig, S3Config,
+        EndpointOverride, Endpoints, GithubReleaseConfig, S3Config, collect_config,
     };
-    use crate::Config;
 
     #[allow(clippy::result_large_err)]
     #[test]

src/common.rs

@@ -3,7 +3,7 @@ use std::ops::Deref;
 
 use actix_http::{Payload, StatusCode};
 use actix_web::{FromRequest, HttpRequest, ResponseError};
-use futures_util::future::{ready, Ready};
+use futures_util::future::{Ready, ready};
 use thiserror::Error;
 
 /// `IntelPath` represents a URL-encoded path which is safe to use both

src/intel_path.rs

@@ -10,7 +10,7 @@
 
 use std::sync::Arc;
 
-use actix_web::{guard, web, App, HttpServer};
+use actix_web::{App, HttpServer, guard, web};
 use prometheus::{Encoder, TextEncoder};
 use reqwest::{Client, ClientBuilder};
 use tokio::sync::mpsc::channel;
@@ -19,7 +19,7 @@ use tracing_actix_web::TracingLogger;
 use tracing_bunyan_formatter::{BunyanFormattingLayer, JsonStorageLayer};
 use tracing_log::LogTracer;
 use tracing_subscriber::layer::SubscriberExt;
-use tracing_subscriber::{fmt, EnvFilter, Registry};
+use tracing_subscriber::{EnvFilter, Registry, fmt};
 
 use artifacts::download_artifacts;
 use browse::list;
@@ -104,14 +104,14 @@ async fn main() {
 
     info!("checking if bucket is available...");
     // check if credentials are set and we have permissions
-    if !config.read_only {
-        if let Err(error) = check_s3(&config.s3).await {
-            warn!(
-                ?error,
-                "s3 storage backend not available, but not running in read-only mode"
-            );
-            // config.read_only = true;
-        }
+    if !config.read_only
+        && let Err(error) = check_s3(&config.s3).await
+    {
+        warn!(
+            ?error,
+            "s3 storage backend not available, but not running in read-only mode"
+        );
+        // config.read_only = true;
     }
 
     info!(?config, "config loaded");

src/main.rs

@@ -1,19 +1,20 @@
-use std::future::Future;
-
-use actix_web::body::BoxBody;
+use actix_web::body::MessageBody;
 use actix_web::dev::{Service, ServiceRequest, ServiceResponse};
-use actix_web::http::header::HeaderName;
 use actix_web::http::Method;
+use actix_web::http::header::HeaderName;
 use actix_web::web;
+use futures_util::future::LocalBoxFuture;
 
 use crate::IntelMission;
 
-pub fn queue_length<S>(
+pub fn queue_length<S, B>(
     req: ServiceRequest,
     srv: &S,
-) -> impl Future<Output = Result<ServiceResponse, actix_web::Error>>
+) -> LocalBoxFuture<'static, Result<ServiceResponse<B>, actix_web::Error>>
 where
-    S: Service<ServiceRequest, Response = ServiceResponse<BoxBody>, Error = actix_web::Error>,
+    S: Service<ServiceRequest, Response = ServiceResponse<B>, Error = actix_web::Error>,
+    S::Future: 'static,
+    B: MessageBody + 'static,
 {
     let length = (req.method() == Method::GET).then(|| {
         let mission = req
@@ -23,19 +24,18 @@ where
         mission.metrics.task_in_queue.get()
     });
     let fut = srv.call(req);
-    async move {
-        let mut resp: ServiceResponse<_> = fut.await?;
+
+    Box::pin(async move {
+        let mut resp = fut.await?;
         if let Some(length) = length {
             // Rewrite the response to return the current counts.
             resp.headers_mut().append(
                 HeaderName::from_static("x-intel-queue-length"),
                 length.into(),
             );
-            return Ok(resp);
         }
-
         Ok(resp)
-    }
+    })
 }
 
 #[cfg(test)]
@@ -44,13 +44,13 @@ mod tests {
 
     use actix_http::Request;
     use actix_web::dev::{Service, ServiceResponse};
-    use actix_web::test::{call_service, init_service, TestRequest};
-    use actix_web::{web, App};
+    use actix_web::test::{TestRequest, call_service, init_service};
+    use actix_web::{App, web};
     use reqwest::Client;
 
     use crate::common::S3Config;
     use crate::storage::get_anonymous_s3_client;
-    use crate::{queue_length, IntelMission, Metrics};
+    use crate::{IntelMission, Metrics, queue_length};
 
     async fn make_service(
         metrics: Arc<Metrics>,

src/queue.rs

@@ -1,13 +1,14 @@
 use actix_web::http::{Method, Uri};
-use actix_web::{guard, web, HttpResponse, Route};
+use actix_web::{HttpResponse, Route, guard, web};
 use lazy_static::lazy_static;
 use regex::Regex;
 
 use crate::error::Result;
 use crate::intel_path::IntelPath;
 use crate::{
+    Error,
     common::{Config, Endpoints, IntelMission, IntelResponse, Redirect, Task},
-    utils, Error,
+    utils,
 };
 
 pub fn simple_intel(
@@ -444,19 +445,19 @@ pub async fn index(path: IntelPath, config: web::Data<Config>) -> IntelResponse
 mod tests {
     use std::sync::Arc;
 
-    use actix_http::{body, Request};
+    use actix_http::{Request, body};
+    use actix_web::App;
     use actix_web::dev::{Service, ServiceResponse};
     use actix_web::http::StatusCode;
-    use actix_web::test::{call_service, init_service, TestRequest};
-    use actix_web::App;
+    use actix_web::test::{TestRequest, call_service, init_service};
+    use figment::Figment;
     use figment::providers::{Format, Toml};
     use figment::util::map;
-    use figment::Figment;
     use httpmock::MockServer;
     use reqwest::ClientBuilder;
     use rstest::rstest;
     use serial_test::serial;
-    use tokio::sync::mpsc::{channel, Receiver};
+    use tokio::sync::mpsc::{Receiver, channel};
     use url::Url;
 
     use crate::common::EndpointOverride;

src/repos.rs

@@ -1,8 +1,8 @@
 //! S3 storage backend.
 use std::time::Duration;
 
-use aws_sdk_s3::config::{BehaviorVersion, Region};
 use aws_sdk_s3::Client as S3Client;
+use aws_sdk_s3::config::Region;
 use tokio::time::timeout;
 
 use crate::common::S3Config;