diff --git a/.github/workflows/build-releases.yml b/.github/workflows/build-releases.yml
index 98b9dfc0d..c19766b18 100644
--- a/.github/workflows/build-releases.yml
+++ b/.github/workflows/build-releases.yml
@@ -19,10 +19,8 @@ jobs:
         ghcr.io/mastodon/mastodon
       # Do not use cache when building releases, so apt update is always ran and the release always contain the latest packages
       cache: false
-      # Only tag with latest when ran against the latest stable branch
-      # This needs to be updated after each minor version release
       flavor: |
-        latest=${{ startsWith(github.ref, 'refs/tags/v4.1.') }}
+        latest=false
       tags: |
         type=pep440,pattern={{raw}}
         type=pep440,pattern=v{{major}}.{{minor}}