Update .forgejo/workflows/pgo-binary.yml

revert even more pgo

.forgejo/workflows/docker.yml

@@ -0,0 +1,58 @@
+# .
+name: Build and Push Multi-Arch Container
+
+env:
+  REGISTRY_HOST: codeberg.org
+  REGISTRY_REPO: gigirassy/riptide
+
+on:
+  push:
+
+
+jobs:
+  build:
+    runs-on: self-hosted
+    
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY_HOST }}
+          username: ${{ secrets.REGISTRY_USER }}
+          password: ${{ secrets.REGISTRY_PASS }}
+
+      - name: Build image (export tar)
+        run: |
+          docker buildx build \
+            --platform linux/amd64 \
+            --output type=docker,dest=image.tar \
+            -t local-image:scan \
+            .
+
+      - name: Scan image with Grype
+        run: |
+          docker run --rm \
+            -v $(pwd):/work \
+            anchore/grype:latest \
+            docker-archive:/work/image.tar \
+            --fail-on high \
+            -o table
+
+      - name: Build and push multi-arch image
+        if: success()
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          push: true
+          platforms: linux/amd64, linux/arm64
+          tags: |
+            ${{ env.REGISTRY_HOST }}/${{ env.REGISTRY_REPO }}:latest

.forgejo/workflows/pgo-binary.yml

@@ -0,0 +1,279 @@
+name: Build and Release
+
+on:
+  push:
+    tags:
+      - "0.*"
+
+env:
+  LINUX_TARGET: x86_64-unknown-linux-musl
+  PGO_SITEMAP: https://riptide-pgo.blitzw.in/sitemaps/sitemap.xml
+
+jobs:
+  build-linux:
+    name: Build Linux musl binary with PGO
+    runs-on: self-hosted
+
+    container:
+      image: rust:alpine
+
+    steps:
+      - name: Install node
+        run: |
+          apk add --no-cache \
+            nodejs
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Install dependencies
+        run: |
+          apk add --no-cache \
+            build-base \
+            musl-dev \
+            curl \
+            file \
+            binutils \
+            coreutils
+
+      - name: Install Rust target and LLVM tools
+        run: |
+          rustup target add "$LINUX_TARGET"
+          rustup component add llvm-tools-preview
+
+      - name: Locate LLVM tools
+        id: llvm
+        shell: sh
+        run: |
+          SYSROOT="$(rustc --print sysroot)"
+          HOST="$(rustc -vV | sed -n 's/^host: //p')"
+          LLVM_BIN="$SYSROOT/lib/rustlib/$HOST/bin"
+
+          test -x "$LLVM_BIN/llvm-profdata"
+
+          echo "LLVM_BIN=$LLVM_BIN" >> "$GITHUB_OUTPUT"
+
+          "$LLVM_BIN/llvm-profdata" --version
+          rustc --version
+          rustc -vV
+
+      - name: Build instrumented binary
+        shell: sh
+        run: |
+          set -eu
+
+          rm -rf /tmp/riptide-pgo
+          mkdir -p /tmp/riptide-pgo/profiles
+
+          RUSTFLAGS="
+            -C target-feature=+crt-static
+            -C profile-generate=/tmp/riptide-pgo/profiles
+          " cargo build \
+            --release \
+            --target "$LINUX_TARGET"
+
+          cp \
+            "target/$LINUX_TARGET/release/riptide" \
+            /tmp/riptide-pgo/riptide
+
+          chmod +x /tmp/riptide-pgo/riptide
+
+          file /tmp/riptide-pgo/riptide
+
+      - name: Verify instrumented binary
+        shell: sh
+        run: |
+          /tmp/riptide-pgo/riptide --help
+
+      - name: Train PGO profile
+        shell: sh
+        env:
+          LLVM_PROFILE_FILE: /tmp/riptide-pgo/profiles/%m-%p.profraw
+        run: |
+          set -eu
+
+          SITEMAP_URL='https://riptide-pgo.blitzw.in/sitemaps/sitemap.xml'
+          SITEMAP_URL_2='https://riptide-pgo.blitzw.in/sitemaps2/sitemap.xml'
+
+          printf 'Training round 1 with sitemap: <%s>\n' "$SITEMAP_URL"
+
+          timeout 10m \
+          /tmp/riptide-pgo/riptide \
+          --sitemap "$SITEMAP_URL" \
+          --output /tmp/pgo-urls.txt \
+          --scheme https \
+          --workers 16 \
+          --timeout-secs 30
+
+          test -s /tmp/pgo-urls.txt
+
+          echo "Training round 1 input:"
+          wc -l /tmp/pgo-urls.txt
+
+          printf 'Training round 2 with sitemap: <%s>\n' "$SITEMAP_URL_2"
+
+          timeout 10m \
+          /tmp/riptide-pgo/riptide \
+          --sitemap "$SITEMAP_URL_2" \
+          --output /tmp/pgo-urls-2.txt \
+          --scheme https \
+          --workers 16 \
+          --timeout-secs 30
+
+          test -s /tmp/pgo-urls-2.txt
+
+          echo "Training round 2 input:"
+          wc -l /tmp/pgo-urls-2.txt
+
+          echo "Generated profiles:"
+          find /tmp/riptide-pgo/profiles \
+            -name '*.profraw' \
+            -type f \
+            -print
+
+          test -n "$(
+            find /tmp/riptide-pgo/profiles \
+              -name '*.profraw' \
+              -type f \
+              -print -quit
+                )"
+
+      - name: Merge Round 1 of PGO profiles
+        shell: sh
+        run: |
+          set -eu
+
+          LLVM_PROFDATA="${{ steps.llvm.outputs.LLVM_BIN }}/llvm-profdata"
+
+          "$LLVM_PROFDATA" merge \
+            -o /tmp/riptide-pgo.profdata \
+            /tmp/riptide-pgo/profiles/*.profraw
+
+          test -s /tmp/riptide-pgo.profdata
+
+
+      - name: Build finalized Linux binary
+        shell: sh
+        run: |
+          set -eu
+
+          RUSTFLAGS="
+            -C target-feature=+crt-static
+            -C profile-use=/tmp/riptide-pgo.profdata
+          " cargo build \
+            --release \
+            --target "$LINUX_TARGET"
+
+          mkdir -p dist
+
+          cp \
+            "target/$LINUX_TARGET/release/riptide" \
+            dist/riptide
+
+          chmod +x dist/riptide
+
+      - name: Verify static binary
+        shell: sh
+        run: |
+          file dist/riptide
+
+          if readelf -l dist/riptide | grep -q 'INTERP'; then
+            echo "ERROR: binary has a dynamic ELF interpreter"
+            exit 1
+          fi
+
+      - name: Smoke test
+        shell: sh
+        run: |
+          ./dist/riptide --help
+
+      - name: Upload Linux binary
+        uses: https://code.forgejo.org/forgejo/upload-artifact@v4
+        with:
+          name: riptide-linux
+          path: dist/riptide
+
+  release:
+    name: Create Forgejo release
+    runs-on: self-hosted
+    needs:
+      - build-linux
+
+    container:
+      image: alpine:latest
+
+    steps:
+      - name: Install release tools
+        run: |
+          apk add --no-cache \
+            curl \
+            jq \
+            coreutils \
+            nodejs
+
+      - name: Download Linux artifact
+        uses: forgejo/download-artifact@v5
+        with:
+          name: riptide-linux
+          path: release
+
+      - name: Calculate checksums
+        id: checksum
+        run: |
+          LINUX_SHA="$(sha256sum release/riptide | awk '{print $1}')"
+
+          echo "LINUX_SHA=$LINUX_SHA" >> "$GITHUB_OUTPUT"
+
+      - name: Create Forgejo release
+        env:
+          FORGEJO_TOKEN: ${{ secrets.FORGEJO_TOKEN }}
+          REPO: ${{ github.repository }}
+          TAG: ${{ github.ref_name }}
+          LINUX_SHA: ${{ steps.checksum.outputs.LINUX_SHA }}
+        run: |
+          cat > release.json <<EOF
+          {
+            "tag_name": "$TAG",
+            "name": "$TAG",
+            "body": "initial release\nriptide: static linux musl binary (x86_64).\n$LINUX_SHA  riptide",
+            "draft": false,
+            "prerelease": false
+          }
+          EOF
+
+          curl -fsSL \
+            -X POST \
+            -H "Authorization: token $FORGEJO_TOKEN" \
+            -H "Content-Type: application/json" \
+            "https://codeberg.org/api/v1/repos/$REPO/releases" \
+            --data-binary @release.json
+
+      - name: Get release ID
+        id: release
+        env:
+          FORGEJO_TOKEN: ${{ secrets.FORGEJO_TOKEN }}
+          REPO: ${{ github.repository }}
+          TAG: ${{ github.ref_name }}
+        run: |
+          RELEASE_ID="$(
+            curl -fsSL \
+              -H "Authorization: token $FORGEJO_TOKEN" \
+              "https://codeberg.org/api/v1/repos/$REPO/releases/tags/$TAG" |
+              jq -r '.id'
+          )"
+
+          test "$RELEASE_ID" != "null"
+          echo "id=$RELEASE_ID" >> "$GITHUB_OUTPUT"
+
+      - name: Upload Linux binary
+        env:
+          FORGEJO_TOKEN: ${{ secrets.FORGEJO_TOKEN }}
+          REPO: ${{ github.repository }}
+          RELEASE_ID: ${{ steps.release.outputs.id }}
+        run: |
+          curl -fsSL \
+            -X POST \
+            -H "Authorization: token $FORGEJO_TOKEN" \
+            -H "Content-Type: application/octet-stream" \
+            --data-binary @release/riptide \
+            "https://codeberg.org/api/v1/repos/$REPO/releases/$RELEASE_ID/assets?name=riptide"

.woodpecker/release.yml

@@ -1,22 +0,0 @@
-when:
-  event: [tag]
-
-steps:
-  - name: build-static
-    image: kgrv/rust:latest
-    commands:
-      - RUSTFLAGS="-Ctarget-feature=+crt-static" compile --release
-      - cp target/*/release/riptide riptide/
-      - tar -czf riptide.tgz -C riptide .
-
-  - name: release-tag
-    image: woodpeckerci/plugin-release
-    settings:
-      base-url: https://codeberg.org
-      api-key:
-        from_secret: RIPTIDE_TOK
-      title: ${CI_COMMIT_TAG}
-      prerelease: false
-      checksum: sha256
-      files:
-        - riptide.tgz

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "riptide"
-version = "0.1.0"
+version = "0.2.1"
 edition = "2021"
 
 [profile.release]

Dockerfile

@@ -0,0 +1,34 @@
+ARG RIPTIDE_TAG=0.1.0
+
+FROM rust:alpine AS build
+
+ARG TARGETARCH
+ARG RIPTIDE_TAG
+
+ENV RUSTFLAGS="-C target-feature=+crt-static"
+
+RUN apk add --no-cache build-base musl-dev pkgconf git
+
+RUN case "$TARGETARCH" in \
+      amd64) cargo_target=x86_64-unknown-linux-musl ;; \
+      arm64) cargo_target=aarch64-unknown-linux-musl ;; \
+      *) exit 1 ;; \
+    esac && \
+    rustup target add "$cargo_target" && \
+    cargo install \
+      --git https://codeberg.org/gigirassy/riptide \
+      --tag "$RIPTIDE_TAG" \
+      --root /out \
+      --target "$cargo_target"
+
+FROM kgrv/mini:tn
+
+COPY --from=build --chown=mini:mini /out/bin/riptide /riptide
+
+VOLUME ["/data"]
+
+WORKDIR /data
+
+ENTRYPOINT ["tini", "--", "/riptide"]
+
+CMD []

README.md

@@ -2,13 +2,39 @@
 
 quick, recursive, sitemap url extractor
 
+riptide, with 16 workers, can rip through a 121,000+ link sitemap in less than 6 seconds.
+
+```
+nune@nunes-pc ~/sitemaps2> time riptide \
+                                                              --sitemap https://www.************.com/sitemap.axd \
+                                                              --output ****.txt \
+                                                              --scheme https \
+                                                              --workers 16 \
+________________________________________________________
+Executed in    5.99 secs      fish           external
+   usr time  528.67 millis    1.14 millis  527.53 millis
+   sys time  201.95 millis    0.21 millis  201.74 millis
+```
+
 ## installation and usage:
 
-install rust, cargo and git. then install riptide with...
+### install (linux static binary, x86_64)
+
+*this assumes /usr/local/bin is part of your path, youll need sha256sum*
+
+*concerned about what the script does? check the url!*
+
+``curl https://codeberg.org/gigirassy/riptide/raw/branch/main/misc/riptideinstaller.sh | bash``
+
+### install (cargo)
 
 ``cargo install --git https://codeberg.org/gigirassy/riptide/``
 
-then in your shell...
+
+
+### usage
+
+**linux/bsd**
 
 ```bash
 riptide \
@@ -19,6 +45,13 @@ riptide \
                                --user-agent "Mozilla/5.0 (X11; Linux i686; rv:151.0) Gecko/20100101 Firefox/151.0"
 ```
 
+**windows**
+in same directory as riptide.
+
+```bash
+.\riptide.exe --sitemap https://example.com/sitemap.xml --output examplecomurls.txt --scheme https --workers 16 --user-agent "Mozilla/5.0 (X11; Linux i686; rv:151.0) Gecko/20100101 Firefox/151.0"
+```
+
 for searching for terms in urls after output is completed, i recommend [ripgrep](https://github.com/BurntSushi/ripgrep).
 
 by default, riptide identifies itself in user agent, the above example overrides it.

misc/riptideinstaller.sh

@@ -0,0 +1,92 @@
+
+#!/bin/sh
+set -eu
+
+repo="gigirassy/riptide"
+base="https://codeberg.org/$repo"
+api="https://codeberg.org/api/v1/repos/$repo/releases/latest"
+
+green="$(printf '\033[1;32m')"
+red="$(printf '\033[1;31m')"
+blue="$(printf '\033[1;34m')"
+reset="$(printf '\033[0m')"
+
+info() {
+    printf '%s[+]%s %s\n' "$green" "$reset" "$*"
+}
+
+step() {
+    printf '%s[*]%s %s\n' "$blue" "$reset" "$*"
+}
+
+error() {
+    printf '%s[-]%s %s\n' "$red" "$reset" "$*" >&2
+}
+
+for cmd in curl jq sha256sum grep awk sed; do
+    command -v "$cmd" >/dev/null 2>&1 || {
+        error "missing dependency: $cmd !"
+        exit 1
+    }
+done
+
+tmpdir="$(mktemp -d)"
+trap 'rm -rf "$tmpdir"' EXIT
+
+if command -v run0 >/dev/null 2>&1; then
+    asroot="run0"
+elif command -v doas >/dev/null 2>&1; then
+    asroot="doas"
+elif command -v sudo >/dev/null 2>&1; then
+    asroot="sudo"
+else
+    error "run0, doas, or sudo is required"
+    exit 1
+fi
+
+step "fetching latest release information..."
+
+release_json="$(curl -fsSL "$api")"
+
+version="$(printf '%s' "$release_json" | jq -r '.tag_name')"
+[ -n "$version" ] && [ "$version" != "null" ] || {
+    error "failed to determine latest version!"
+    exit 1
+}
+
+checksum="$(
+    printf '%s' "$release_json" |
+    jq -r '.body' |
+    grep -oE '[a-f0-9]{64}' |
+    head -n1
+)"
+
+[ -n "$checksum" ] || {
+    error "failed to locate checksum!"
+    exit 1
+}
+
+url="$base/releases/download/$version/riptide"
+
+info "latest version is $version!"
+info "checksum is $checksum!"
+
+step "downloading riptide..."
+
+curl -fsSL "$url" -o "$tmpdir/riptide"
+
+step "verifying checksum..."
+
+printf '%s  %s\n' "$checksum" "$tmpdir/riptide" | sha256sum -c - >/dev/null
+
+info "checksum verified!"
+
+chmod 755 "$tmpdir/riptide"
+
+step "installing to /usr/local/bin/riptide"
+step "authentication may be required ($asroot)"
+
+
+"$asroot" install -m 755 "$tmpdir/riptide" /usr/local/bin/riptide
+
+info "riptide $version installed successfully <3 have a nice day"

src/main.rs

@@ -2,7 +2,6 @@ use std::{
     collections::{HashSet, VecDeque},
     fs,
     io::Read,
-    sync::Arc,
     time::Duration,
 };
 
@@ -11,7 +10,7 @@ use clap::{Parser, ValueEnum};
 use flate2::read::GzDecoder;
 use futures::stream::{self, StreamExt};
 use mimalloc::MiMalloc;
-use reqwest::{header, Client};
+use reqwest::{Client};
 use roxmltree::Document;
 use tokio::time::timeout;
 use url::Url;