Update streamly to the latest master

.github/workflows/haskell.yml

@@ -15,7 +15,7 @@ jobs:
       # ------------------------------------------------------------------------
       CABAL_REINIT_CONFIG: y
       LC_ALL: C.UTF-8
-      DISABLE_SDIST_BUILD: n
+      DISABLE_SDIST_BUILD: ${{ matrix.disable_sdist_build }}
 
       # ------------------------------------------------------------------------
       # cabal options
@@ -42,8 +42,6 @@ jobs:
       fail-fast: false
       matrix:
         name:
-          - 8.2.2
-          - 8.4.4
           - 8.6.5
           - 8.8.4
           - 8.10.7+fusion-plugin
@@ -54,54 +52,57 @@ jobs:
 
         include:
 
-          - name: 8.2.2
-            ghc_version: 8.2.2
-            runner: ubuntu-latest
-            cabal_version: 3.2
-
-          - name: 8.4.4
-            ghc_version: 8.4.4
-            runner: ubuntu-latest
-            cabal_version: 3.2
-
           - name: 8.6.5
             ghc_version: 8.6.5
             runner: ubuntu-latest
-            cabal_version: 3.2
+            cabal_version: 3.2.0.0
+            cabal_project: cabal.project.user
 
           - name: 8.8.4
             ghc_version: 8.8.4
             runner: ubuntu-latest
-            cabal_version: 3.2
+            cabal_version: 3.2.0.0
+            cabal_project: cabal.project.user
 
           - name: 8.10.7+fusion-plugin
             ghc_version: 8.10.7
             pack_options: CABAL_BUILD_OPTIONS="-f fusion-plugin"
             runner: ubuntu-latest
-            cabal_version: 3.2
+            cabal_version: 3.2.0.0
+            cabal_project: cabal.project.user
 
           - name: 9.0.1+werror
             ghc_version: 9.0.1
             runner: ubuntu-latest
             cabal_project: cabal.project.Werror
-            cabal_version: 3.2
+            cabal_version: 3.2.0.0
 
           - name: hlint
             ghc_version: 8.8.4
             pack_options: HLINT_OPTIONS="lint" HLINT_TARGETS="src"
             runner: ubuntu-latest
-            cabal_version: 3.2
+            cabal_version: 3.2.0.0
+            cabal_project: cabal.project.user
 
           - name: 8.10.7+macos
             ghc_version: 8.10.7
             runner: macos-latest
-            cabal_version: 3.2
+            cabal_version: 3.2.0.0
+            cabal_project: cabal.project.user
 
           - name: 9.2.1
             ghc_version: 9.2.1
             runner: ubuntu-latest
-            cabal_version: 3.6
+            cabal_version: 3.6.0.0
             pack_options: DISABLE_BENCH=y
+            cabal_project: cabal.project.user
+
+          - name: 9.4.2
+            ghc_version: 9.4.2
+            runner: ubuntu-latest
+            cabal_project: cabal.project.user
+            cabal_version: 3.8.1.0
+            disable_sdist_build: "y"
 
     steps:
     - uses: actions/checkout@v2

.packcheck.ignore

@@ -4,5 +4,6 @@ stack.yaml
 appveyor.yml
 .gitignore
 cabal.project.Werror
+cabal.project.user
 default.nix
 cabal.project

appveyor.yml

@@ -60,7 +60,7 @@ environment:
     # If you have not committed packcheck.sh in your repo at PACKCHECK_LOCAL_PATH
     # then it is automatically pulled from this URL.
     PACKCHECK_GITHUB_URL: "https://raw.githubusercontent.com/composewell/packcheck"
-    PACKCHECK_GITHUB_COMMIT: "v0.6.0"
+    PACKCHECK_GITHUB_COMMIT: "35efa99b2082d13722b8a0183ac6455df98e91b9"
 
     # Override the temp directory to avoid sed escaping issues
     # See https://github.com/haskell/cabal/issues/5386

benchmark/Main.hs

@@ -9,17 +9,17 @@
 module Main (main) where
 
 import Control.Monad (unless)
-import Data.Semigroup (cycle1)
-import Data.Word (Word8)
 import Data.Function ((&))
-import Streamly.Internal.Data.Array.Foreign (Array)
-import Streamly.Internal.Data.Stream.IsStream.Type (fromStreamD, toStreamD)
-import Streamly.Prelude (SerialT)
+import Data.Word (Word8)
+import Streamly.Data.Array (Array)
+import Streamly.Data.Stream (Stream)
 import System.Directory (getCurrentDirectory, doesFileExist)
 import System.Environment (lookupEnv)
 
-import qualified Streamly.Internal.Data.Stream.IsStream as Stream
-import qualified Streamly.Internal.Data.Array.Foreign as Array
+import qualified Streamly.Data.Array as Array
+import qualified Streamly.Data.Fold as Fold
+import qualified Streamly.Data.Stream as Stream
+import qualified Streamly.Internal.Data.Stream as Stream (parseD)
 import qualified Streamly.Internal.FileSystem.File as File
 import qualified Streamly.Internal.LZ4 as LZ4
 import qualified Streamly.Internal.LZ4.Config as LZ4
@@ -76,11 +76,11 @@ bootstrap fp = do
     fileExists <- doesFileExist normalizedFp
     unless fileExists $ do
         putStrLn $ "Normalizing " ++ fp
-        let fileStream = Stream.unfold File.read fp
+        let fileStream = Stream.unfold File.reader (cycle fp)
             combinedStream =
-                Stream.arraysOf _64KB
+                Stream.foldMany (Array.writeN _64KB)
                     $ Stream.take _10MB
-                    $ cycle1 fileStream
+                    $ fileStream
         combinedStream & File.fromChunks normalizedFp
         combinedStream & LZ4.compressChunks LZ4.defaultBlockConfig 65537
                        & File.fromChunks compressedFpBig
@@ -97,10 +97,13 @@ bootstrap fp = do
             headerList = magicLE ++ [flg, bd, headerChk]
             header = Stream.fromList headerList
         headerArr <- Stream.fold (Array.writeN (length headerList)) header
-        (bf, ff) <- Stream.parseD LZ4.simpleFrameParserD header
-        combinedStream & compressChunksFrame bf ff 65537
-                       & Stream.cons headerArr
-                       & File.fromChunks compressedWith
+        x0 <- Stream.parseD LZ4.simpleFrameParserD header
+        case x0 of
+            Right (bf, ff) ->
+                combinedStream & compressChunksFrame bf ff 65537
+                            & Stream.cons headerArr
+                            & File.fromChunks compressedWith
+            Left _ -> return ()
 
     where
 
@@ -112,29 +115,29 @@ bootstrap fp = do
         :: LZ4.BlockConfig
         -> LZ4.FrameConfig
         -> Int
-        -> Stream.SerialT IO (Array.Array Word8)
-        -> Stream.SerialT IO (Array.Array Word8)
+        -> Stream IO (Array.Array Word8)
+        -> Stream IO (Array.Array Word8)
     compressChunksFrame bf ff i_ strm =
         if LZ4.hasEndMark ff
-        then (fromStreamD . LZ4.compressChunksD bf i_ . toStreamD) strm
+        then (LZ4.compressChunksD bf i_) strm
                  `Stream.append` Stream.fromPure endMarkArr
-        else (fromStreamD . LZ4.compressChunksD bf i_ . toStreamD) strm
+        else (LZ4.compressChunksD bf i_) strm
 
 
 --------------------------------------------------------------------------------
 -- Benchmark helpers
 --------------------------------------------------------------------------------
 
-type Combinator = SerialT IO (Array Word8) -> SerialT IO (Array Word8)
+type Combinator = Stream IO (Array Word8) -> Stream IO (Array Word8)
 
 {-# INLINE benchCorpus #-}
 benchCorpus :: Int -> String -> String -> Combinator -> Benchmark
 benchCorpus bufsize name corpus combinator =
     let bname = ("bufsize(" ++ show bufsize ++ ")/" ++ name ++ "/" ++ corpus)
      in bench bname $ nfIO $ do
-            Stream.unfold File.readChunksWithBufferOf (bufsize, corpus)
+            Stream.unfold File.chunkReaderWith (bufsize, corpus)
                 & combinator
-                & Stream.drain
+                & Stream.fold Fold.drain
 
 --------------------------------------------------------------------------------
 -- Benchmarks
@@ -156,16 +159,14 @@ decompress bufsize corpus =
 decompressWith :: Int -> String -> Benchmark
 decompressWith bufsize corpus =
     benchCorpus bufsize "decompressWith" corpus
-        $ fromStreamD
-        . LZ4.decompressChunksWithD LZ4.simpleFrameParserD . toStreamD
+        $ LZ4.decompressChunksWithD LZ4.simpleFrameParserD
 
 {-# INLINE resize #-}
 resize :: Int -> String -> Benchmark
 resize bufsize corpus =
     benchCorpus bufsize "resize" corpus
-        $ fromStreamD
-        . LZ4.resizeChunksD LZ4.defaultBlockConfig LZ4.defaultFrameConfig
-        . toStreamD
+        $ LZ4.resizeChunksD LZ4.defaultBlockConfig LZ4.defaultFrameConfig
+
 
 --------------------------------------------------------------------------------
 -- Reading environment

cabal.project.Werror

@@ -2,3 +2,14 @@ packages: streamly-lz4.cabal
 
 package streamly-lz4
   ghc-options: -Werror
+
+source-repository-package
+   type: git
+   location: https://github.com/composewell/streamly.git
+   tag: master
+
+source-repository-package
+  type: git
+  location: https://github.com/composewell/streamly.git
+  tag: master
+  subdir: core

cabal.project.user

@@ -0,0 +1,14 @@
+packages: .
+
+source-repository-package
+   type: git
+   location: https://github.com/composewell/streamly.git
+   tag: master
+
+source-repository-package
+  type: git
+  location: https://github.com/composewell/streamly.git
+  tag: master
+  subdir: core
+
+jobs: 1

default.nix

@@ -1,91 +1,46 @@
 # CAUTION! a spelling mistake in arg string is ignored silently.
 #
-# To use ghc-8.6.5
-# nix-shell --argstr compiler "ghc865"
+# To use ghc-8.10.7
+# nix-shell --argstr compiler "ghc8107"
 
 {
   nixpkgs ?
-    import (builtins.fetchTarball https://github.com/NixOS/nixpkgs/archive/refs/tags/21.05.tar.gz)
+    import (builtins.fetchTarball
+      https://github.com/NixOS/nixpkgs/archive/refs/tags/22.05.tar.gz)
         {}
-, compiler ? "default"
-, c2nix ? "" # cabal2nix CLI options
-# TODO
-#, sources ? [] # e.g. [./. ./benchmark]
-#, hdeps ? [] # e.g. [time, mtl]
-#, deps ? [] # e.g. [SDL2]
+, compiler ? "ghc922"
 }:
-let haskellPackages =
-        if compiler == "default"
-        then nixpkgs.haskellPackages
-        else nixpkgs.haskell.packages.${compiler};
-
-    # we can possibly avoid adding our package to HaskellPackages like
-    # in the case of nix-shell for a single package?
-    mkPackage = super: pkg: path: opts: inShell:
-                let orig = super.callCabal2nixWithOptions pkg path opts {};
-                 in if inShell
-                    # Avoid copying the source directory to nix store by using
-                    # src = null.
-                    then orig.overrideAttrs (oldAttrs: { src = null; })
-                    else orig;
-
-    flags = "--benchmark --flag fusion-plugin" + " " + c2nix;
+let
+    utils =
+      let src = fetchGit {
+            url = "[email protected]:composewell/composewell-haskell.git";
+            ref = "update-some";
+          }; in (import "${src}/utils.nix") { inherit nixpkgs; };
+
+
+    haskellPackages =
+      let src = fetchGit {
+            url = "[email protected]:composewell/composewell-haskell.git";
+            ref = "update-some";
+          }; in (import "${src}/haskellPackages.nix")
+            { inherit nixpkgs;
+              inherit compiler; };
 
     mkHaskellPackages = inShell:
-        haskellPackages.override {
-            overrides = self: super:
-                with nixpkgs.haskell.lib;
-                {
-                    streamly-lz4 = mkPackage super "streamly-lz4" ./. flags inShell;
-
-                    streamly =
-                      nixpkgs.haskell.lib.overrideCabal
-                        (super.callHackageDirect
-                          { pkg = "streamly";
-                            ver = "0.8.2";
-                            sha256 = "0jhsdd71kqw0k0aszg1qb1l0wbxl1r73hsmkdgch4vlx43snlc8a";
-                          } {})
-                        (old:
-                          { librarySystemDepends =
-                              if builtins.currentSystem == "x86_64-darwin"
-                              then [nixpkgs.darwin.apple_sdk.frameworks.Cocoa]
-                              else [];
-                            enableLibraryProfiling = false;
-                            doHaddock = false;
-                          });
-
-                    unicode-data =
-                      super.callHackageDirect
-                        { pkg = "unicode-data";
-                          ver = "0.3.0";
-                          sha256 = "0izxxk7qgq22ammzmwc4cs4nlhzp7y55gzyas2a8bzhdpac1j7yx";
-                        } {};
+      haskellPackages.override (old: {
+        overrides =
+          nixpkgs.lib.composeExtensions
+            (old.overrides or (_: _: {}))
+            (with nixpkgs.haskell.lib; self: super: {
+                streamly-lz4 =
+                  utils.local super "streamly-lz4" ./. "--benchmark" inShell;
+            });
+      });
 
-                    tasty-bench =
-                      super.callHackageDirect
-                        { pkg = "tasty-bench";
-                          ver = "0.3";
-                          sha256 = "0na1q52zr8p1zz8hby4242yjr2zma3js4n91avl7ibsa2y51vrc4";
-                        } {};
-                };
-        };
+    shellDrv = mkHaskellPackages true;
 
-    drv = mkHaskellPackages true;
+    shell = utils.mkShell shellDrv (p: [p.streamly-lz4]) true;
 
-    shell = drv.shellFor {
-        packages = p:
-          [ p.streamly-lz4
-          ];
-        doBenchmark = true;
-        # Use a better prompt
-        shellHook = ''
-          export CABAL_DIR="$(pwd)/.cabal.nix"
-          if test -n "$PS_SHELL"
-          then
-            export PS1="$PS_SHELL\[$bldred\](nix)\[$txtrst\] "
-          fi
-        '';
-    };
 in if nixpkgs.lib.inNixShell
    then shell
    else (mkHaskellPackages false).streamly-lz4