Import upstream v0.16.22, stripped
Upstream commit: 474dd0229cb20cf513036619781ed97bd8073c3f Enterprise-only files removed or emptied: 63 Enterprise-only snippets removed: 117 in 50 files Dangling module declarations removed: 5 Cargo edits turning enterprise off: 14 Verification: clean Enterprise feature gates left for rebuilt features: 19 in 18 files Produced by tools/fork/strip.py. The full report is in docs/fork/strip-reports/ on main.
This commit is contained in:
@@ -0,0 +1,64 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Decode a raw Stalwart blob object into its original bytes.
|
||||
|
||||
Stalwart appends a one-byte compression marker to every blob it writes to the
|
||||
blob store (see BlobStore::put_blob in crates/store/src/dispatch/blob.rs):
|
||||
|
||||
0x00 (NONE_MARKER) the preceding bytes are the verbatim payload
|
||||
0xa1 (LZ4_MARKER) the preceding bytes are an lz4_flex block prefixed with a
|
||||
little-endian u32 holding the uncompressed size
|
||||
other a legacy blob stored without a marker; emitted unchanged
|
||||
|
||||
This mirrors the read path, which inspects the last byte, strips the marker and,
|
||||
for LZ4, decompresses the size-prepended block.
|
||||
|
||||
Reads the object from a file (or stdin with "-") and writes the decoded payload
|
||||
to stdout (or to a file with -o).
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
|
||||
NONE_MARKER = 0x00
|
||||
LZ4_MARKER = 0xA1
|
||||
|
||||
|
||||
def decode(data):
|
||||
if not data:
|
||||
return data
|
||||
marker = data[-1]
|
||||
if marker == LZ4_MARKER:
|
||||
import lz4.block
|
||||
return lz4.block.decompress(data[:-1])
|
||||
if marker == NONE_MARKER:
|
||||
return data[:-1]
|
||||
print(f"warning: no known compression marker (last byte 0x{marker:02x}); "
|
||||
"emitting bytes unchanged", file=sys.stderr)
|
||||
return data
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("input", help='raw blob file, or "-" to read from stdin')
|
||||
ap.add_argument("-o", "--output",
|
||||
help="write the decoded payload here (default: stdout)")
|
||||
args = ap.parse_args()
|
||||
|
||||
if args.input == "-":
|
||||
data = sys.stdin.buffer.read()
|
||||
else:
|
||||
with open(args.input, "rb") as fh:
|
||||
data = fh.read()
|
||||
|
||||
out = decode(data)
|
||||
|
||||
if args.output:
|
||||
with open(args.output, "wb") as fh:
|
||||
fh.write(out)
|
||||
else:
|
||||
sys.stdout.buffer.write(out)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,166 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
IMAP Log sanitizer - Extracts and groups IMAP transactions from log files
|
||||
"""
|
||||
|
||||
# SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
#
|
||||
# SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
|
||||
import re
|
||||
import json
|
||||
from collections import defaultdict
|
||||
from datetime import datetime
|
||||
import argparse
|
||||
|
||||
def unescape_imap_content(content):
|
||||
|
||||
if content.startswith('"') and content.endswith('"'):
|
||||
content = content[1:-1]
|
||||
|
||||
replacements = {
|
||||
'\\r\\n': '\r\n',
|
||||
'\\n': '\n',
|
||||
'\\r': '\r',
|
||||
'\\t': '\t',
|
||||
'\\"': '"',
|
||||
'\\\\': '\\'
|
||||
}
|
||||
|
||||
for escaped, unescaped in replacements.items():
|
||||
content = content.replace(escaped, unescaped)
|
||||
|
||||
return content
|
||||
|
||||
def parse_imap_log_line(line):
|
||||
|
||||
pattern = r'(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z)\s+TRACE\s+Raw IMAP\s+(input received|output sent)\s+.*?remoteIp\s*=\s*([^,]+),\s*remotePort\s*=\s*(\d+).*?contents\s*=\s*(.+)$'
|
||||
|
||||
match = re.search(pattern, line)
|
||||
if not match:
|
||||
return None
|
||||
|
||||
timestamp, direction, remote_ip, remote_port, contents = match.groups()
|
||||
|
||||
return {
|
||||
'timestamp': timestamp,
|
||||
'direction': direction,
|
||||
'remote_ip': remote_ip.strip(),
|
||||
'remote_port': int(remote_port),
|
||||
'contents': unescape_imap_content(contents.strip()),
|
||||
'raw_line': line.strip()
|
||||
}
|
||||
|
||||
def group_by_connection(log_entries):
|
||||
|
||||
connections = defaultdict(list)
|
||||
|
||||
for entry in log_entries:
|
||||
if entry:
|
||||
key = f"{entry['remote_ip']}:{entry['remote_port']}"
|
||||
connections[key].append(entry)
|
||||
|
||||
for key in connections:
|
||||
connections[key].sort(key=lambda x: x['timestamp'])
|
||||
|
||||
return dict(connections)
|
||||
|
||||
def format_imap_transaction(entries):
|
||||
|
||||
transaction = []
|
||||
|
||||
for entry in entries:
|
||||
direction_symbol = "C: " if "input received" in entry['direction'] else "S: "
|
||||
timestamp = entry['timestamp']
|
||||
content = entry['contents']
|
||||
|
||||
if content.endswith('\\r\\n') or content.endswith('\r\n'):
|
||||
content = content.rstrip('\\r\\n\r\n')
|
||||
|
||||
transaction.append(f"[{timestamp}] {direction_symbol}{content}")
|
||||
|
||||
return transaction
|
||||
|
||||
def write_output_file(connections, output_file):
|
||||
|
||||
with open(output_file, 'w', encoding='utf-8') as f:
|
||||
f.write("IMAP Transaction Log Analysis\n")
|
||||
f.write("=" * 50 + "\n\n")
|
||||
|
||||
for connection_key, entries in connections.items():
|
||||
f.write(f"Connection: {connection_key}\n")
|
||||
f.write("-" * 30 + "\n")
|
||||
f.write(f"Total messages: {len(entries)}\n")
|
||||
f.write(f"Duration: {entries[0]['timestamp']} to {entries[-1]['timestamp']}\n\n")
|
||||
|
||||
transaction = format_imap_transaction(entries)
|
||||
for line in transaction:
|
||||
f.write(line + "\n")
|
||||
|
||||
f.write("\n" + "=" * 50 + "\n\n")
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Parse IMAP log files and group transactions by connection')
|
||||
parser.add_argument('input_file', help='Input log file path')
|
||||
parser.add_argument('-o', '--output', default='imap_transactions.txt',
|
||||
help='Output file path (default: imap_transactions.txt)')
|
||||
parser.add_argument('-j', '--json', action='store_true',
|
||||
help='Also output raw data as JSON')
|
||||
parser.add_argument('-v', '--verbose', action='store_true',
|
||||
help='Enable verbose output')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.verbose:
|
||||
print(f"Reading log file: {args.input_file}")
|
||||
|
||||
log_entries = []
|
||||
imap_line_count = 0
|
||||
|
||||
try:
|
||||
with open(args.input_file, 'r', encoding='utf-8') as f:
|
||||
for line_num, line in enumerate(f, 1):
|
||||
if 'Raw IMAP' in line:
|
||||
imap_line_count += 1
|
||||
parsed_entry = parse_imap_log_line(line)
|
||||
if parsed_entry:
|
||||
log_entries.append(parsed_entry)
|
||||
elif args.verbose:
|
||||
print(f"Warning: Could not parse line {line_num}: {line.strip()}")
|
||||
|
||||
except FileNotFoundError:
|
||||
print(f"Error: File '{args.input_file}' not found")
|
||||
return 1
|
||||
except Exception as e:
|
||||
print(f"Error reading file: {e}")
|
||||
return 1
|
||||
|
||||
if args.verbose:
|
||||
print(f"Found {imap_line_count} Raw IMAP lines")
|
||||
print(f"Successfully parsed {len(log_entries)} entries")
|
||||
|
||||
connections = group_by_connection(log_entries)
|
||||
|
||||
if args.verbose:
|
||||
print(f"Found {len(connections)} unique connections:")
|
||||
for conn_key, entries in connections.items():
|
||||
print(f" {conn_key}: {len(entries)} messages")
|
||||
|
||||
try:
|
||||
write_output_file(connections, args.output)
|
||||
print(f"IMAP transactions written to: {args.output}")
|
||||
|
||||
if args.json:
|
||||
json_file = args.output.rsplit('.', 1)[0] + '.json'
|
||||
with open(json_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(connections, f, indent=2, ensure_ascii=False)
|
||||
print(f"Raw data written to: {json_file}")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error writing output: {e}")
|
||||
return 1
|
||||
|
||||
return 0
|
||||
|
||||
if __name__ == "__main__":
|
||||
exit(main())
|
||||
@@ -0,0 +1,156 @@
|
||||
#!/usr/bin/env python3
|
||||
"""List the S3 object keys of every blob that is still actively linked.
|
||||
|
||||
Stalwart reference-counts blobs through link entries kept in the data store
|
||||
under the SUBSPACE_BLOB_LINK subspace (the PostgreSQL table named "k"). A blob
|
||||
is garbage-collectable once it has no surviving link; this tool reports the
|
||||
opposite set: the S3 keys of blobs that are still referenced, so they can be
|
||||
diffed against the actual contents of an S3 bucket to find orphans.
|
||||
|
||||
Key layout in table "k" (the subspace byte is NOT stored, it only selects the
|
||||
table). All integers are big-endian:
|
||||
|
||||
32 bytes Commit : <hash:32> (blob exists marker)
|
||||
40 bytes Id link : <hash:32><id:8>
|
||||
41 bytes Doc link: <hash:32><account_id:4><collection:1><document_id:4>
|
||||
44 bytes Tmp link: <hash:32><account_id:4><until:8> (active iff until > now)
|
||||
|
||||
A blob is active when it has at least one Id/Doc link, or a Temporary link whose
|
||||
"until" (unix seconds) is still in the future. The 32-byte Commit marker alone
|
||||
does not keep a blob alive.
|
||||
|
||||
The S3 object key is the optional configured key prefix (literal string)
|
||||
followed by the custom-base32 encoding of the 32-byte blob hash, matching
|
||||
S3Store::build_key in crates/store/src/backend/s3/mod.rs.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
BLOB_HASH_LEN = 32
|
||||
ID_LINK = BLOB_HASH_LEN + 8 # 40
|
||||
DOC_LINK = BLOB_HASH_LEN + 8 + 1 # 41
|
||||
TEMP_LINK = BLOB_HASH_LEN + 4 + 8 # 44
|
||||
|
||||
BASE32_ALPHABET = b"abcdefghijklmnopqrstuvwxyz792013"
|
||||
|
||||
|
||||
class Base32Writer:
|
||||
"""Faithful port of utils::codec::base32_custom::Base32Writer."""
|
||||
|
||||
def __init__(self, prefix=""):
|
||||
self.last_byte = 0
|
||||
self.pos = 0
|
||||
self.out = [prefix] if prefix else []
|
||||
|
||||
def _push_byte(self, byte, is_remainder):
|
||||
p = self.pos % 5
|
||||
if p == 0:
|
||||
ch1 = (byte & 0xF8) >> 3
|
||||
ch2 = 0xFF
|
||||
elif p == 1:
|
||||
ch1 = ((self.last_byte & 0x07) << 2) | ((byte & 0xC0) >> 6)
|
||||
ch2 = (byte & 0x3E) >> 1
|
||||
elif p == 2:
|
||||
ch1 = ((self.last_byte & 0x01) << 4) | ((byte & 0xF0) >> 4)
|
||||
ch2 = 0xFF
|
||||
elif p == 3:
|
||||
ch1 = ((self.last_byte & 0x0F) << 1) | (byte >> 7)
|
||||
ch2 = (byte & 0x7C) >> 2
|
||||
else:
|
||||
ch1 = ((self.last_byte & 0x03) << 3) | ((byte & 0xE0) >> 5)
|
||||
ch2 = byte & 0x1F
|
||||
|
||||
self.out.append(chr(BASE32_ALPHABET[ch1]))
|
||||
if not is_remainder:
|
||||
if ch2 != 0xFF:
|
||||
self.out.append(chr(BASE32_ALPHABET[ch2]))
|
||||
self.last_byte = byte
|
||||
self.pos += 1
|
||||
|
||||
def write(self, data):
|
||||
for byte in data:
|
||||
self._push_byte(byte, False)
|
||||
return self
|
||||
|
||||
def finalize(self):
|
||||
if self.pos % 5 != 0:
|
||||
self._push_byte(0, True)
|
||||
return "".join(self.out)
|
||||
|
||||
|
||||
def s3_key(blob_hash, prefix=""):
|
||||
return Base32Writer(prefix).write(blob_hash).finalize()
|
||||
|
||||
|
||||
def be_u64(b):
|
||||
return int.from_bytes(b, "big")
|
||||
|
||||
|
||||
def collect_active_hashes(rows, now, include_expired_temporary=False):
|
||||
active = set()
|
||||
unknown = 0
|
||||
for (key,) in rows:
|
||||
key = bytes(key)
|
||||
n = len(key)
|
||||
if n == BLOB_HASH_LEN:
|
||||
continue
|
||||
if n in (ID_LINK, DOC_LINK):
|
||||
active.add(key[:BLOB_HASH_LEN])
|
||||
elif n == TEMP_LINK:
|
||||
until = be_u64(key[BLOB_HASH_LEN + 4:BLOB_HASH_LEN + 12])
|
||||
if include_expired_temporary or until > now:
|
||||
active.add(key[:BLOB_HASH_LEN])
|
||||
else:
|
||||
unknown += 1
|
||||
if unknown:
|
||||
print(f"warning: skipped {unknown} key(s) of unexpected length",
|
||||
file=sys.stderr)
|
||||
return active
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("--host", default=os.environ.get("PGHOST", "localhost"))
|
||||
ap.add_argument("--port", type=int, default=int(os.environ.get("PGPORT", "5432")))
|
||||
ap.add_argument("--user", default=os.environ.get("PGUSER", "stalwart"))
|
||||
ap.add_argument("--password", default=os.environ.get("PGPASSWORD", "stalwart"))
|
||||
ap.add_argument("--dbname", default=os.environ.get("PGDATABASE", "stalwart"))
|
||||
ap.add_argument("--table", default="k",
|
||||
help="SUBSPACE_BLOB_LINK table name (default: k)")
|
||||
ap.add_argument("--prefix", default="",
|
||||
help="S3 key_prefix configured on the blob store (default: none)")
|
||||
ap.add_argument("--now", type=int, default=None,
|
||||
help="override unix-seconds used to expire temporary links")
|
||||
ap.add_argument("--include-expired-temporary", action="store_true",
|
||||
help="treat expired temporary links as active too")
|
||||
args = ap.parse_args()
|
||||
|
||||
try:
|
||||
import psycopg2
|
||||
conn = psycopg2.connect(host=args.host, port=args.port, user=args.user,
|
||||
password=args.password, dbname=args.dbname)
|
||||
except ImportError:
|
||||
import psycopg
|
||||
conn = psycopg.connect(host=args.host, port=args.port, user=args.user,
|
||||
password=args.password, dbname=args.dbname)
|
||||
|
||||
now = args.now if args.now is not None else int(time.time())
|
||||
|
||||
with conn, conn.cursor() as cur:
|
||||
cur.execute(f'SELECT k FROM "{args.table}"')
|
||||
rows = cur.fetchall()
|
||||
conn.close()
|
||||
|
||||
active = collect_active_hashes(rows, now,
|
||||
include_expired_temporary=args.include_expired_temporary)
|
||||
|
||||
for h in sorted(active):
|
||||
print(s3_key(h, args.prefix))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
File diff suppressed because it is too large
Load Diff
Executable
+70
@@ -0,0 +1,70 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Minify a self-contained HTML file (with inlined <style> and <script>).
|
||||
#
|
||||
# Uses html-minifier-terser via `npx`, which runs it from npx's cache without
|
||||
# touching the repo's package.json / node_modules. The first invocation will
|
||||
# download the package; subsequent runs are instant.
|
||||
#
|
||||
# Usage:
|
||||
# resources/scripts/minify_html.sh [--gzip] path/to/file.html
|
||||
#
|
||||
# Writes `path/to/file.ext.min` next to the source, and with --gzip also
|
||||
# `path/to/file.ext.min.gz` for pages that are served pre-compressed.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
gzip_output=0
|
||||
if [[ "${1:-}" == "--gzip" ]]; then
|
||||
gzip_output=1
|
||||
shift
|
||||
fi
|
||||
|
||||
if [[ $# -ne 1 ]]; then
|
||||
echo "usage: $(basename "$0") [--gzip] <file.html>" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
src="$1"
|
||||
|
||||
if [[ ! -f "$src" ]]; then
|
||||
echo "error: not a file: $src" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v npx >/dev/null 2>&1; then
|
||||
echo "error: npx not found in PATH (install Node.js)" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# login.html -> login.min.html
|
||||
dir=$(dirname -- "$src")
|
||||
base=$(basename -- "$src")
|
||||
stem="${base%.*}"
|
||||
ext="${base##*.}"
|
||||
dst="$dir/$stem.$ext.min"
|
||||
|
||||
npx -y html-minifier-terser@latest \
|
||||
--collapse-whitespace \
|
||||
--conservative-collapse \
|
||||
--remove-comments \
|
||||
--minify-css true \
|
||||
--minify-js true \
|
||||
--decode-entities \
|
||||
-o "$dst" \
|
||||
"$src"
|
||||
|
||||
before=$(wc -c < "$src" | tr -d ' ')
|
||||
after=$(wc -c < "$dst" | tr -d ' ')
|
||||
saved=$((before - after))
|
||||
pct=$(awk "BEGIN { printf \"%.1f\", ($saved / $before) * 100 }")
|
||||
|
||||
echo "$src -> $dst"
|
||||
echo " $before B -> $after B ($saved B, $pct% smaller)"
|
||||
|
||||
if [ "$gzip_output" = "1" ]; then
|
||||
gzip -9 -n -c "$dst" > "$dst.gz"
|
||||
gz=$(wc -c < "$dst.gz" | tr -d ' ')
|
||||
gzpct=$(awk "BEGIN { printf \"%.1f\", (($before - $gz) / $before) * 100 }")
|
||||
echo " gzip: $dst.gz $gz B ($gzpct% smaller than source)"
|
||||
fi
|
||||
@@ -0,0 +1,269 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Stalwart SEL code remover
|
||||
|
||||
This script removes SEL code from the Stalwart codebase by:
|
||||
1. Removing entire .rs files that contain "SPDX-License-Identifier: LicenseRef-SEL" in their first comment
|
||||
2. Removing SEL snippets marked with SPDX-SnippetBegin/End from mixed files
|
||||
|
||||
Usage: python ossify.py <stalwart_repository>/crates
|
||||
"""
|
||||
|
||||
# SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
#
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
import os
|
||||
import sys
|
||||
import re
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
from typing import List, Tuple, Optional
|
||||
|
||||
def find_first_comment_block(content: str) -> Optional[str]:
|
||||
|
||||
lines = content.strip().split('\n')
|
||||
|
||||
if not lines:
|
||||
return None
|
||||
|
||||
first_line = lines[0].strip()
|
||||
|
||||
if first_line.startswith('/*'):
|
||||
comment_lines = []
|
||||
in_comment = True
|
||||
|
||||
for line in lines:
|
||||
if in_comment:
|
||||
comment_lines.append(line)
|
||||
if '*/' in line:
|
||||
break
|
||||
|
||||
return '\n'.join(comment_lines)
|
||||
|
||||
elif first_line.startswith('//'):
|
||||
comment_lines = []
|
||||
|
||||
for line in lines:
|
||||
stripped = line.strip()
|
||||
if stripped.startswith('//'):
|
||||
comment_lines.append(line)
|
||||
elif stripped == '':
|
||||
comment_lines.append(line)
|
||||
else:
|
||||
break
|
||||
|
||||
return '\n'.join(comment_lines)
|
||||
|
||||
return None
|
||||
|
||||
def should_remove_file(file_path: str) -> bool:
|
||||
|
||||
try:
|
||||
with open(file_path, 'r', encoding='utf-8') as f:
|
||||
content = f.read()
|
||||
|
||||
first_comment = find_first_comment_block(content)
|
||||
if first_comment and 'SPDX-License-Identifier: LicenseRef-SEL' in first_comment:
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error reading file {file_path}: {e}")
|
||||
|
||||
return False
|
||||
|
||||
CRATE_ROOT_STUB = """/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
"""
|
||||
|
||||
def is_crate_root(file_path: str) -> bool:
|
||||
|
||||
path = Path(file_path)
|
||||
|
||||
return (
|
||||
path.name == 'lib.rs'
|
||||
and path.parent.name == 'src'
|
||||
and (path.parent.parent / 'Cargo.toml').is_file()
|
||||
)
|
||||
|
||||
def remove_proprietary_snippets(content: str) -> Tuple[str, int]:
|
||||
|
||||
snippets_removed = 0
|
||||
|
||||
lines = content.split('\n')
|
||||
result_lines = []
|
||||
i = 0
|
||||
|
||||
while i < len(lines):
|
||||
line = lines[i]
|
||||
|
||||
if '// SPDX-SnippetBegin' in line:
|
||||
|
||||
snippet_start = i
|
||||
snippet_lines = []
|
||||
j = i
|
||||
|
||||
while j < len(lines):
|
||||
snippet_lines.append(lines[j])
|
||||
if '// SPDX-SnippetEnd' in lines[j]:
|
||||
break
|
||||
j += 1
|
||||
|
||||
snippet_content = '\n'.join(snippet_lines)
|
||||
if 'SPDX-License-Identifier: LicenseRef-SEL' in snippet_content:
|
||||
|
||||
snippets_removed += 1
|
||||
i = j + 1
|
||||
continue
|
||||
else:
|
||||
|
||||
result_lines.append(line)
|
||||
i += 1
|
||||
else:
|
||||
result_lines.append(line)
|
||||
i += 1
|
||||
|
||||
return '\n'.join(result_lines), snippets_removed
|
||||
|
||||
def process_rust_file(file_path: str, dry_run: bool = False) -> dict:
|
||||
|
||||
result = {
|
||||
'file': file_path,
|
||||
'action': 'none',
|
||||
'snippets_removed': 0,
|
||||
'error': None
|
||||
}
|
||||
|
||||
try:
|
||||
|
||||
if should_remove_file(file_path):
|
||||
if is_crate_root(file_path):
|
||||
result['action'] = 'file_emptied'
|
||||
if not dry_run:
|
||||
with open(file_path, 'w', encoding='utf-8') as f:
|
||||
f.write(CRATE_ROOT_STUB)
|
||||
else:
|
||||
result['action'] = 'file_removed'
|
||||
if not dry_run:
|
||||
os.remove(file_path)
|
||||
return result
|
||||
|
||||
with open(file_path, 'r', encoding='utf-8') as f:
|
||||
original_content = f.read()
|
||||
|
||||
modified_content, snippets_removed = remove_proprietary_snippets(original_content)
|
||||
|
||||
if snippets_removed > 0:
|
||||
result['action'] = 'snippets_removed'
|
||||
result['snippets_removed'] = snippets_removed
|
||||
|
||||
if not dry_run:
|
||||
with open(file_path, 'w', encoding='utf-8') as f:
|
||||
f.write(modified_content)
|
||||
|
||||
except Exception as e:
|
||||
result['error'] = str(e)
|
||||
|
||||
return result
|
||||
|
||||
def find_rust_files(directory: str) -> List[str]:
|
||||
|
||||
rust_files = []
|
||||
|
||||
for root, dirs, files in os.walk(directory):
|
||||
for file in files:
|
||||
if file.endswith('.rs'):
|
||||
rust_files.append(os.path.join(root, file))
|
||||
|
||||
return rust_files
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Remove Enterprise licensed code from Stalwart codebase'
|
||||
)
|
||||
parser.add_argument(
|
||||
'directory',
|
||||
help='Directory containing Stalwart code to process'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--dry-run',
|
||||
action='store_true',
|
||||
help='Show what would be done without making changes'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--verbose',
|
||||
action='store_true',
|
||||
help='Show detailed output for each file'
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if not os.path.isdir(args.directory):
|
||||
print(f"Error: {args.directory} is not a valid directory")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"Processing Rust files in: {args.directory}")
|
||||
if args.dry_run:
|
||||
print("DRY RUN MODE - No changes will be made")
|
||||
print()
|
||||
|
||||
rust_files = find_rust_files(args.directory)
|
||||
|
||||
if not rust_files:
|
||||
print("No .rs files found in the specified directory")
|
||||
return
|
||||
|
||||
print(f"Found {len(rust_files)} Rust files")
|
||||
print()
|
||||
|
||||
files_removed = 0
|
||||
files_emptied = 0
|
||||
files_with_snippets_removed = 0
|
||||
total_snippets_removed = 0
|
||||
errors = []
|
||||
|
||||
for file_path in rust_files:
|
||||
result = process_rust_file(file_path, args.dry_run)
|
||||
|
||||
if result['error']:
|
||||
errors.append(f"{file_path}: {result['error']}")
|
||||
continue
|
||||
|
||||
if result['action'] == 'file_removed':
|
||||
files_removed += 1
|
||||
if args.verbose or args.dry_run:
|
||||
action_text = "Would remove" if args.dry_run else "Removed"
|
||||
print(f"{action_text} file: {file_path}")
|
||||
|
||||
elif result['action'] == 'file_emptied':
|
||||
files_emptied += 1
|
||||
if args.verbose or args.dry_run:
|
||||
action_text = "Would empty" if args.dry_run else "Emptied"
|
||||
print(f"{action_text} crate root: {file_path}")
|
||||
|
||||
elif result['action'] == 'snippets_removed':
|
||||
files_with_snippets_removed += 1
|
||||
total_snippets_removed += result['snippets_removed']
|
||||
if args.verbose or args.dry_run:
|
||||
action_text = "Would remove" if args.dry_run else "Removed"
|
||||
print(f"{action_text} {result['snippets_removed']} snippet(s) from: {file_path}")
|
||||
|
||||
print("\nSummary:")
|
||||
action_text = "Would be" if args.dry_run else "Were"
|
||||
print(f"- {files_removed} files {action_text.lower()} completely removed")
|
||||
print(f"- {files_emptied} crate roots {action_text.lower()} emptied")
|
||||
print(f"- {total_snippets_removed} proprietary snippets {action_text.lower()} removed from {files_with_snippets_removed} files")
|
||||
|
||||
if errors:
|
||||
print(f"- {len(errors)} errors occurred:")
|
||||
for error in errors:
|
||||
print(f" {error}")
|
||||
|
||||
if args.dry_run:
|
||||
print("\nRun without --dry-run to apply changes")
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,413 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Repair orphaned value chunks in a Stalwart FoundationDB data store.
|
||||
|
||||
Values larger than 100000 bytes are split across a base key and one key per
|
||||
extra chunk, suffixed with a single byte: `key`, `key || 0x00`, `key || 0x01`,
|
||||
and so on. Releases up to and including v0.16.18 overwrote such a value without
|
||||
removing the chunk keys the previous, longer value had used. The leftovers are
|
||||
spliced onto later reads, which surfaces as:
|
||||
|
||||
ERROR Data corruption detected (store.data-corruption)
|
||||
details = 'Archive integrity compromised'
|
||||
|
||||
This script finds those leftovers and deletes them. It is read-only unless
|
||||
--commit is given.
|
||||
|
||||
Only subspaces whose keys are prefix-free are scanned, so that a key one byte
|
||||
longer than its predecessor is unambiguously a chunk and never a distinct
|
||||
record. This is a stricter condition than `is_chunked_subspace` in
|
||||
crates/store/src/backend/foundationdb/mod.rs, which answers "can chunking
|
||||
happen here" rather than "are keys here prefix-free". See CHUNKED_SUBSPACES
|
||||
below for what is excluded and why. Keys are additionally checked against the
|
||||
shapes a real record can have in their subspace, so a legal record is never
|
||||
mistaken for a chunk.
|
||||
|
||||
Two cases are handled:
|
||||
|
||||
* The base value is under 100000 bytes, so the record is no longer chunked
|
||||
and every chunk key that follows it is stale. Unambiguous.
|
||||
|
||||
* The base value is exactly 100000 bytes, so the record is still chunked and
|
||||
the live chunk count has to be worked out. Two independent methods must
|
||||
agree before anything is deleted: chunk sizes, since every chunk of a live
|
||||
value except the last is exactly 100000 bytes, and the trailing xxh3
|
||||
checksum that Stalwart archives carry. Records where they disagree, or
|
||||
where neither applies, are left untouched and reported.
|
||||
|
||||
Each repair re-reads the record and deletes inside a single transaction, so a
|
||||
record rewritten between the scan and the delete is judged on what is actually
|
||||
stored rather than on what the scan saw.
|
||||
|
||||
Requirements:
|
||||
pip install foundationdb xxhash
|
||||
|
||||
The `foundationdb` package version must match the cluster's API version.
|
||||
|
||||
Usage:
|
||||
python repair_fdb_chunks.py --cluster-file /etc/foundationdb/fdb.cluster
|
||||
python repair_fdb_chunks.py --cluster-file /etc/foundationdb/fdb.cluster --commit
|
||||
|
||||
Stop Stalwart before running with --commit.
|
||||
"""
|
||||
|
||||
# SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
#
|
||||
# SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
|
||||
try:
|
||||
import fdb
|
||||
except ImportError:
|
||||
sys.exit("Missing dependency: pip install foundationdb")
|
||||
|
||||
try:
|
||||
import xxhash
|
||||
except ImportError:
|
||||
sys.exit("Missing dependency: pip install xxhash")
|
||||
|
||||
MAX_VALUE_SIZE = 100000
|
||||
|
||||
MAGIC_MARKER = 1 << 7
|
||||
VERSIONED = 1 << 6
|
||||
HASHED = 1 << 5
|
||||
|
||||
U32_LEN = 4
|
||||
U64_LEN = 8
|
||||
|
||||
CHUNKED_SUBSPACES = {
|
||||
b"p": "property",
|
||||
b"e": "queue message",
|
||||
b"f": "task queue",
|
||||
b"d": "directory",
|
||||
b"s": "registry",
|
||||
b"j": "deleted items",
|
||||
b"w": "spam samples",
|
||||
b"r": "inbound reports",
|
||||
b"h": "outbound reports",
|
||||
b"o": "telemetry spans",
|
||||
}
|
||||
|
||||
BASE_KEY_LENGTHS = {
|
||||
b"e": {9},
|
||||
b"f": {17},
|
||||
b"o": {9, 11},
|
||||
b"d": {11},
|
||||
b"s": {11},
|
||||
b"j": {11},
|
||||
b"w": {11},
|
||||
b"r": {11},
|
||||
b"h": {11},
|
||||
}
|
||||
|
||||
# EmailField::Threading, the only property indexed by hash
|
||||
THREADING_FIELD = 90
|
||||
|
||||
|
||||
def is_record_key(subspace, key):
|
||||
lengths = BASE_KEY_LENGTHS.get(subspace)
|
||||
if lengths is not None:
|
||||
return len(key) in lengths
|
||||
if subspace == b"p":
|
||||
# Property (11), IndexProperty::Integer (19), the 2-byte schema version written as
|
||||
# ValueClass::Any, and IndexProperty::Hash whose CheekyHash is 1 to 16 bytes
|
||||
if len(key) in (2, 11, 19):
|
||||
return True
|
||||
return 12 <= len(key) <= 27 and key[6] == THREADING_FIELD
|
||||
return False
|
||||
|
||||
READ_BATCH = 32
|
||||
|
||||
|
||||
def is_valid_archive(value):
|
||||
"""Mirror of validate_marker_and_contents in crates/store/src/write/serialize.rs."""
|
||||
if not value:
|
||||
return False
|
||||
|
||||
marker = value[-1]
|
||||
if marker & MAGIC_MARKER == 0:
|
||||
return False
|
||||
|
||||
contents = value[:-1]
|
||||
|
||||
if marker & VERSIONED != 0:
|
||||
if len(contents) < U64_LEN + U32_LEN:
|
||||
return False
|
||||
contents = contents[:-U64_LEN]
|
||||
elif marker & HASHED != 0:
|
||||
if len(contents) < U32_LEN:
|
||||
return False
|
||||
else:
|
||||
# Unversioned archives carry no checksum, so length cannot be verified
|
||||
return False
|
||||
|
||||
contents, archive_hash = contents[:-U32_LEN], contents[-U32_LEN:]
|
||||
hash32 = xxhash.xxh3_64_intdigest(contents) & 0xFFFFFFFF
|
||||
return hash32.to_bytes(U32_LEN, "big") == archive_hash
|
||||
|
||||
|
||||
def count_by_chunk_size(chunk_values):
|
||||
"""Live chunk count implied by chunk sizes, or None if it cannot be read off them.
|
||||
|
||||
`chunk_value` splits with value.chunks(MAX_VALUE_SIZE), so every chunk of the live value
|
||||
except its last is exactly MAX_VALUE_SIZE bytes. The first short chunk therefore ends the
|
||||
live value. This never undercounts: if the live value happens to be an exact multiple of
|
||||
MAX_VALUE_SIZE the first short chunk belongs to the stale tail, which yields a count that
|
||||
is too high and so only leaves orphans behind. It works for every value format.
|
||||
"""
|
||||
for index, chunk in enumerate(chunk_values):
|
||||
if len(chunk) < MAX_VALUE_SIZE:
|
||||
return index + 1
|
||||
return None
|
||||
|
||||
|
||||
def count_by_archive(base_value, chunk_values):
|
||||
"""Live chunk count implied by the archive checksum, or None if undecidable.
|
||||
|
||||
Refuses when more than one prefix validates rather than taking the shortest, so a value
|
||||
that embeds a complete inner archive cannot cause the live tail to be truncated.
|
||||
"""
|
||||
matches = []
|
||||
if is_valid_archive(base_value):
|
||||
matches.append(0)
|
||||
|
||||
accumulated = bytearray(base_value)
|
||||
for count, chunk in enumerate(chunk_values, start=1):
|
||||
accumulated.extend(chunk)
|
||||
if is_valid_archive(bytes(accumulated)):
|
||||
matches.append(count)
|
||||
|
||||
return matches[0] if len(matches) == 1 else None
|
||||
|
||||
|
||||
def live_chunk_count(base_value, chunk_values):
|
||||
"""Number of chunk keys that belong to the current value, or None if undecidable.
|
||||
|
||||
Both independent methods must agree. The size rule never undercounts and the checksum
|
||||
rule is format specific, so requiring agreement means a disagreement leaves the record
|
||||
untouched instead of guessing at an operator's data.
|
||||
"""
|
||||
if len(base_value) < MAX_VALUE_SIZE:
|
||||
return 0
|
||||
|
||||
by_size = count_by_chunk_size(chunk_values)
|
||||
by_archive = count_by_archive(base_value, chunk_values)
|
||||
|
||||
if by_size is not None and by_archive is not None:
|
||||
return by_size if by_size == by_archive else None
|
||||
|
||||
# Non-archive formats (pickled tasks and traces, raw thread indexes, untrusted archives)
|
||||
# carry no checksum, so the size rule is the only available signal
|
||||
return by_size if by_archive is None else None
|
||||
|
||||
|
||||
TRANSACTION_TOO_OLD = 1007
|
||||
TRANSACTION_TIMED_OUT = 1031
|
||||
|
||||
|
||||
def read_range(db, begin, end, limit):
|
||||
"""Read one batch starting at `begin`, returning the rows and the batch size that worked.
|
||||
|
||||
A chunked record holds 100000 bytes per key, so a batch can carry several megabytes and
|
||||
outlive FoundationDB's five second transaction window. Every attempt starts a fresh
|
||||
transaction from `begin`, so a retry resumes exactly where the failed one started, and the
|
||||
batch is halved on each timeout until it fits.
|
||||
"""
|
||||
while True:
|
||||
tr = db.create_transaction()
|
||||
try:
|
||||
rows = list(
|
||||
tr.get_range(begin, end, limit=limit, streaming_mode=fdb.StreamingMode.want_all)
|
||||
)
|
||||
return rows, limit
|
||||
except fdb.FDBError as err:
|
||||
if err.code in (TRANSACTION_TOO_OLD, TRANSACTION_TIMED_OUT) and limit > 1:
|
||||
limit = max(1, limit // 2)
|
||||
print(f" transaction exceeded its time budget, retrying with batch of {limit}")
|
||||
continue
|
||||
tr.on_error(err).wait()
|
||||
|
||||
|
||||
def repair_record(db, subspace, base_key):
|
||||
"""Re-read one record and delete its orphans in a single transaction.
|
||||
|
||||
The scan reads the store in many separate transactions, so a record's base value and its
|
||||
chunks are not a consistent snapshot. Recomputing inside the transaction that performs
|
||||
the delete removes that race: if the record was rewritten in the meantime the deletion is
|
||||
based on what is actually there, not on what the scan saw.
|
||||
"""
|
||||
while True:
|
||||
tr = db.create_transaction()
|
||||
try:
|
||||
rows = [
|
||||
(bytes(kv.key), bytes(kv.value))
|
||||
for kv in tr.get_range(
|
||||
base_key,
|
||||
base_key + bytes([0xFF]),
|
||||
streaming_mode=fdb.StreamingMode.want_all,
|
||||
)
|
||||
]
|
||||
if not rows or rows[0][0] != base_key:
|
||||
return 0
|
||||
|
||||
base_value = rows[0][1]
|
||||
chunks = [
|
||||
(key, value)
|
||||
for key, value in rows[1:]
|
||||
if len(key) == len(base_key) + 1
|
||||
and key.startswith(base_key)
|
||||
and not is_record_key(subspace, key)
|
||||
]
|
||||
|
||||
count = live_chunk_count(base_value, [value for _, value in chunks])
|
||||
if count is None:
|
||||
return 0
|
||||
|
||||
orphans = [key for key, _ in chunks[count:]]
|
||||
for key in orphans:
|
||||
tr.clear(key)
|
||||
tr.commit().wait()
|
||||
return len(orphans)
|
||||
except fdb.FDBError as err:
|
||||
tr.on_error(err).wait()
|
||||
|
||||
|
||||
class Stats:
|
||||
def __init__(self):
|
||||
self.records = 0
|
||||
self.orphans = 0
|
||||
self.deleted = 0
|
||||
self.undecidable = 0
|
||||
|
||||
|
||||
def scan_subspace(db, subspace, label, commit, verbose, stats):
|
||||
begin = subspace
|
||||
end = bytes([subspace[0] + 1])
|
||||
|
||||
base_key = None
|
||||
base_value = None
|
||||
chunk_keys = []
|
||||
chunk_values = []
|
||||
cursor = begin
|
||||
|
||||
def flush_record():
|
||||
if base_key is None:
|
||||
return
|
||||
stats.records += 1
|
||||
if not chunk_keys:
|
||||
return
|
||||
|
||||
count = live_chunk_count(base_value, chunk_values)
|
||||
if count is None:
|
||||
stats.undecidable += 1
|
||||
print(
|
||||
f" ! {label}: cannot determine chunk count for {base_key.hex()} "
|
||||
f"({len(chunk_keys)} chunk keys, {len(base_value)} byte base), left untouched"
|
||||
)
|
||||
return
|
||||
|
||||
orphans = chunk_keys[count:]
|
||||
if not orphans:
|
||||
return
|
||||
|
||||
stats.orphans += len(orphans)
|
||||
if verbose:
|
||||
print(
|
||||
f" - {label}: {base_key.hex()} has {len(orphans)} orphaned "
|
||||
f"chunk(s), keeping {count}"
|
||||
)
|
||||
if commit:
|
||||
stats.deleted += repair_record(db, subspace, base_key)
|
||||
|
||||
batch = READ_BATCH
|
||||
while True:
|
||||
rows, batch = read_range(db, cursor, end, batch)
|
||||
if not rows:
|
||||
break
|
||||
|
||||
for kv in rows:
|
||||
key = bytes(kv.key)
|
||||
value = bytes(kv.value)
|
||||
|
||||
if (
|
||||
base_key is not None
|
||||
and len(key) == len(base_key) + 1
|
||||
and key.startswith(base_key)
|
||||
and not is_record_key(subspace, key)
|
||||
):
|
||||
chunk_keys.append(key)
|
||||
chunk_values.append(value)
|
||||
continue
|
||||
|
||||
flush_record()
|
||||
base_key = key
|
||||
base_value = value
|
||||
chunk_keys = []
|
||||
chunk_values = []
|
||||
|
||||
cursor = bytes(rows[-1].key) + b"\x00"
|
||||
|
||||
flush_record()
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Repair orphaned value chunks in a Stalwart FoundationDB store"
|
||||
)
|
||||
parser.add_argument("--cluster-file", help="Path to fdb.cluster")
|
||||
parser.add_argument(
|
||||
"--api-version", type=int, default=740, help="FoundationDB API version (default: 740)"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--commit",
|
||||
action="store_true",
|
||||
help="Delete the orphaned chunks. Without this the script only reports.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--subspace",
|
||||
action="append",
|
||||
help="Limit the scan to this subspace letter. May be repeated.",
|
||||
)
|
||||
parser.add_argument("--verbose", action="store_true", help="Print every affected record")
|
||||
args = parser.parse_args()
|
||||
|
||||
fdb.api_version(args.api_version)
|
||||
db = fdb.open(args.cluster_file)
|
||||
|
||||
selected = CHUNKED_SUBSPACES
|
||||
if args.subspace:
|
||||
wanted = {s.encode() for s in args.subspace}
|
||||
unknown = wanted - set(CHUNKED_SUBSPACES)
|
||||
if unknown:
|
||||
sys.exit(
|
||||
"Not a chunked subspace: "
|
||||
+ ", ".join(sorted(u.decode() for u in unknown))
|
||||
+ "\nValid: "
|
||||
+ ", ".join(sorted(s.decode() for s in CHUNKED_SUBSPACES))
|
||||
)
|
||||
selected = {s: CHUNKED_SUBSPACES[s] for s in wanted}
|
||||
|
||||
if not args.commit:
|
||||
print("DRY RUN. No key will be deleted. Pass --commit to apply the repair.\n")
|
||||
else:
|
||||
print("COMMIT MODE. Orphaned chunks will be deleted. Stalwart must be stopped.\n")
|
||||
|
||||
stats = Stats()
|
||||
for subspace, label in sorted(selected.items()):
|
||||
print(f"Scanning subspace {subspace.decode()!r} ({label})...")
|
||||
scan_subspace(db, subspace, label, args.commit, args.verbose, stats)
|
||||
|
||||
print(f"\nRecords scanned: {stats.records}")
|
||||
print(f"Orphaned chunks: {stats.orphans}")
|
||||
if stats.undecidable:
|
||||
print(f"Undecidable: {stats.undecidable} (reported above, not modified)")
|
||||
if args.commit:
|
||||
print(f"Chunks deleted: {stats.deleted}")
|
||||
elif stats.orphans:
|
||||
print("\nRe-run with --commit to delete them.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user