yuyr/panda-rpki-oss
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

20261016 修复 Manifest failed-fetch 缓存与子树恢复
Some checks failed
ci / rust (push) Failing after 4s
Details
ci / docker (push) Failing after 1m27s
Details
ci / audit (push) Failing after 3s
Details

Browse Source
This commit is contained in:
yuyr
2026-10-16 15:53:40 +08:00
parent 5f46837b61
commit 2d7501db08
28 changed files with 1565 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files

214
tests/support/test_failed_fetch_extended.py Normal file
View File

@@ -0,0 +1,214 @@
"""Signed, multi-level repository mutations and real CLI restarts over both transports."""
import argparse, functools, hashlib, http.server, json, shutil, ssl, subprocess, sys, tempfile, threading
from datetime import timedelta
from pathlib import Path
from cryptography import x509
from cryptography.hazmat.primitives import serialization, hashes
from cryptography.hazmat.primitives.asymmetric import rsa
from cryptography.hazmat.primitives.serialization import Encoding
import generate_repository as g
from test_public_config import free_port, make_rsync_wrapper, read_json, run_checked, write_config
def canonical_ccr(blob):
# producedAt describes artifact generation, not the validated state. Compare
# every other DER byte while normalizing only that top-level CCR timestamp.
def tlv(offset):
tag = blob[offset]; length = blob[offset+1]; start = offset+2
if length & 128:
count = length & 127; length = int.from_bytes(blob[start:start+count], 'big'); start += count
return tag, start, start+length
tag, outer, end = tlv(0); assert tag == 0x30 and end == len(blob)
_, _, oid_end = tlv(outer)
tag, explicit, _ = tlv(oid_end); assert tag == 0xa0
tag, start, end = tlv(explicit); assert tag == 0x30
while start < end:
tag, value, next_field = tlv(start)
if tag == 0x18:
return blob[:value] + b'0'*(next_field-value) + blob[next_field:]
start = next_field
raise AssertionError('CCR producedAt missing')
def run(binary, evidence=None):
proof = {'transports': [], 'binary_sha256': hashlib.sha256(Path(binary).read_bytes()).hexdigest(), 'ccr_comparison': 'all DER bytes except producedAt; CSV byte identical'}
with tempfile.TemporaryDirectory(prefix='panda-failed-fetch-extended-') as temp:
base = Path(temp); fixture = base/'fixture'; port = free_port()
run_checked([sys.executable, str(Path(__file__).with_name('generate_repository.py')), '--output', str(fixture),
'--rrdp-host', '127.0.0.1', '--rrdp-port', str(port), '--rsync-host', '127.0.0.1', '--strict-names'])
g.STRICT_NAMES = True
root_uri = 'rsync://127.0.0.1:873/custom/root/'
child_uri = 'rsync://127.0.0.1:873/custom/child/'
grand_uri = 'rsync://127.0.0.1:873/custom/grand/'
sibling_uri = 'rsync://127.0.0.1:873/custom/sibling/'
notify = f'https://127.0.0.1:{port}/rrdp/notification.xml'
active = base/'active'; active.mkdir(); (active/'rrdp').mkdir()
repository = base/'repository'
requests = []
class Handler(http.server.SimpleHTTPRequestHandler):
def do_GET(self): requests.append(self.path); super().do_GET()
def log_message(self, *_): pass
server = http.server.ThreadingHTTPServer(('127.0.0.1', port), functools.partial(Handler, directory=str(active)))
tls = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
tls.load_cert_chain(fixture/'certs/rrdp-server.pem', fixture/'certs/rrdp-server.key')
server.socket = tls.wrap_socket(server.socket, server_side=True)
thread = threading.Thread(target=server.serve_forever, daemon=True); thread.start()
original = fixture/'cases/baseline-v1/repository'
root = x509.load_der_x509_certificate((original/'ta.cer').read_bytes())
child = x509.load_der_x509_certificate((original/'root/child.cer').read_bytes())
root_key = serialization.load_pem_private_key((fixture/'keys/root.key').read_bytes(), None)
child_key = serialization.load_pem_private_key((fixture/'keys/child.key').read_bytes(), None)
work = base/'signing'; work.mkdir()
def manifest(folder, number, names, issuer, key, ca_uri):
uri = {'root': root_uri, 'child': child_uri, 'grand': grand_uri, 'sibling': sibling_uri}[folder]
ee_key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
ee = g.sign_cert(x509.Name([g.common_name('Failed Fetch Manifest')]), issuer, ee_key.public_key(), key,
x509.random_serial_number(), False, ee_key, key, ip_ext=g.ip_inherit(), as_ext=g.as_resources(None),
signed_object_uri=uri+folder+'.mft', crl_uri=uri+folder+'.crl', ca_issuer_uri=ca_uri)
content = g.manifest_content(number, [(n, (repository/folder/n).read_bytes()) for n in names])
g.write_cms(content, g.OID_MANIFEST_ECONTENT, ee, ee_key, repository/folder/(folder+'.mft'), work)
def publish(index):
session = f'{index:08d}-1111-4111-8111-222222222222'
objects = sorted(p for p in repository.rglob('*') if p.is_file() and p.suffix in {'.mft', '.crl', '.cer', '.roa', '.asa'})
snapshot = (f'<snapshot xmlns="http://www.ripe.net/rpki/rrdp" version="1" session_id="{session}" serial="1">'
+ ''.join(g.publish('rsync://127.0.0.1:873/custom/'+str(p.relative_to(repository)), p) for p in objects)
+ '</snapshot>').encode()
(active/'rrdp/snapshot.xml').write_bytes(snapshot)
(active/'rrdp/notification.xml').write_text(f'<notification xmlns="http://www.ripe.net/rpki/rrdp" version="1" session_id="{session}" serial="1"><snapshot uri="https://127.0.0.1:{port}/rrdp/snapshot.xml" hash="{hashlib.sha256(snapshot).hexdigest()}"/></notification>')
def reset(case):
if repository.exists(): shutil.rmtree(repository)
shutil.copytree(fixture/'cases'/case/'repository', repository)
try:
for transport in ('rsync-only', 'rrdp-then-rsync'):
target = base/transport; target.mkdir(); log = target/'rsync.log'; wrapper = target/'rsync.py'
make_rsync_wrapper(wrapper, fixture, log)
wrapper.write_text(wrapper.read_text().replace(str(original), str(repository)).replace(
'destination.mkdir(parents=True, exist_ok=True)',
'if destination.exists(): shutil.rmtree(destination)\ndestination.mkdir(parents=True, exist_ok=True)'))
wrapper.chmod(0o755)
reset('baseline-v1'); runs = []; initial_vrps = initial_aspas = initial_ccr = None
def execute(name, expected, partial=False, *, stamp='2026-06-01T00:00:00Z', config_change=None, no_child_fetch=False):
nonlocal initial_vrps, initial_aspas, initial_ccr
publish(len(runs)+1)
out = target/name; config = target/'validate.toml'
write_config(config, fixture=fixture, output=out, state=target/'state', ccr=out/'result.ccr', wrapper=wrapper,
mirror=target/'mirror', sync_preference=transport, rrdp_mode='auto', validation_time=stamp,
resource_mode='rfc6487', scope='publication-point')
if config_change: config.write_text(config_change(config.read_text()))
start = len(requests); rsync_start = len(log.read_text().splitlines()) if log.exists() else 0
result = subprocess.run([binary, 'validate', '--config', str(config)], capture_output=True, text=True, timeout=120)
summary = read_json(out/'summary.json')
assert summary['vrps'] == expected, (transport, name, summary, result.stderr[-5000:])
assert summary['resultStatus'] == ('partial' if partial else 'success'), (name, summary, result.stderr[-5000:])
assert result.returncode == (3 if partial else 0), (name, result.returncode, result.stderr[-5000:])
events = []
for line in result.stderr.splitlines():
try: event = json.loads(line)
except ValueError: continue
if event.get('event') == 'failed_fetch_cache_reused': events.append(event['fields'])
cached = summary['failed_fetch_cached']
# Structured cache events also expose the unchanged validity bounds.
assert cached == len(events), (name, cached, events)
rsync = log.read_text().splitlines()[rsync_start:] if log.exists() else []
if no_child_fetch:
if transport == 'rsync-only': assert len(rsync) == 1, (name, rsync)
assert sum(e['repo_sync_phase'] == 'cache_only' for e in events) == max(0, cached-1), events
if name == 'initial':
initial_vrps = (out/'vrps.csv').read_bytes(); initial_aspas = (out/'vaps.csv').read_bytes(); initial_ccr = canonical_ccr((out/'result.ccr').read_bytes())
if name in {'corrupt-object', 'repeat-failure', 'missing-object', 'bad-crl', 'corrupt-parent'}:
assert (out/'vrps.csv').read_bytes() == initial_vrps
assert (out/'vaps.csv').read_bytes() == initial_aspas
assert canonical_ccr((out/'result.ccr').read_bytes()) == initial_ccr
if transport == 'rrdp-then-rsync': assert not rsync, ('unexpected rsync fallback', name, rsync)
runs.append({'name': name, 'vrps': expected, 'status': summary['resultStatus'], 'cached': cached,
'publication_points': summary['publication_points'], 'http_requests': requests[start:], 'rsync_requests': len(rsync),
'verified_at': [p['verified_at'] for p in sorted(events, key=lambda e:e['manifest_rsync_uri'])]})
execute('initial', 1)
good_roa = (repository/'child/valid.roa').read_bytes()
(repository/'child/valid.roa').write_bytes(b'corrupt downloaded object')
execute('corrupt-object', 1, True)
execute('repeat-failure', 1, True, config_change=lambda s:s.replace('parallel-phase2-object-workers = 2', 'parallel-phase2-object-workers = 1'))
assert runs[-1]['verified_at'] == runs[-2]['verified_at'], 'failure refreshed validation time'
execute('policy-changed', 0, True, config_change=lambda s:s.replace('name = true', 'name = false'))
execute('clock-rollback', 0, True, stamp='2026-05-31T00:00:00Z')
execute('expired', 0, True, stamp='2027-12-01T01:00:00Z')
(repository/'child/valid.roa').unlink(); execute('missing-object', 1, True)
(repository/'child/valid.roa').write_bytes(good_roa)
good_crl = (repository/'child/child.crl').read_bytes(); (repository/'child/child.crl').write_bytes(b'bad crl')
execute('bad-crl', 1, True); (repository/'child/child.crl').write_bytes(good_crl)
(repository/'root/root.mft').write_bytes(b'bad parent'); execute('corrupt-parent', 1, True, no_child_fetch=True)
reset('baseline-v2'); execute('recovered', 2)
manifest('child', 3, ['child.crl'], child.subject, child_key, child_uri+'child.cer')
execute('fresh-empty-child', 0)
(repository/'child/child.mft').write_bytes(b'bad empty child'); execute('cached-empty-child', 0, True)
manifest('root', 4, ['root.crl'], root.subject, root_key, root_uri+'ta.cer')
execute('fresh-deleted-child', 0)
(repository/'root/root.mft').write_bytes(b'bad deleted parent'); execute('cached-deleted-child', 0, True, no_child_fetch=True)
# Restore a valid child, then revoke its certificate in a signed root CRL.
shutil.copy2(fixture/'cases/baseline-v2/repository/child/child.mft', repository/'child/child.mft')
manifest('child', 4, ['child.crl', 'valid.roa', 'valid2.roa', 'valid.asa'], child.subject, child_key, child_uri+'child.cer')
manifest('root', 5, ['root.crl', 'child.cer'], root.subject, root_key, root_uri+'ta.cer')
execute('restored-child', 2)
revoked = x509.RevokedCertificateBuilder().serial_number(child.serial_number).revocation_date(g.VALID_FROM+timedelta(days=32)).build()
crl = (x509.CertificateRevocationListBuilder().issuer_name(root.subject).last_update(g.VALID_FROM+timedelta(days=33))
.next_update(g.VALID_TO-timedelta(days=30)).add_extension(x509.AuthorityKeyIdentifier.from_issuer_public_key(root_key.public_key()), False)
.add_extension(x509.CRLNumber(2), False).add_revoked_certificate(revoked).sign(root_key, hashes.SHA256()))
(repository/'root/root.crl').write_bytes(crl.public_bytes(Encoding.DER))
manifest('root', 6, ['root.crl', 'child.cer'], root.subject, root_key, root_uri+'ta.cer'); execute('fresh-revoked-child', 0)
(repository/'root/root.mft').write_bytes(b'bad revoked parent'); execute('cached-revoked-child', 0, True, no_child_fetch=True)
# A genuine root -> child -> grandchild chain, with payload only at depth 2.
(repository/'root/root.crl').write_bytes((original/'root/root.crl').read_bytes())
grand_key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
grand_name = x509.Name([g.common_name('Grandchild CA')])
grand = g.sign_cert(grand_name, child.subject, grand_key.public_key(), child_key, 909, True, grand_key, child_key,
ip_ext=g.ip_resources([g.CHILD_PREFIX]), as_ext=g.as_resources([g.TEST_ASN]),
ca_repository=grand_uri, manifest_uri=grand_uri+'grand.mft', notify_uri=notify,
crl_uri=child_uri+'child.crl', ca_issuer_uri=root_uri+'child.cer')
(repository/'child/grand.cer').write_bytes(grand.public_bytes(Encoding.DER)); (repository/'grand').mkdir()
(repository/'grand/grand.crl').write_bytes(g.make_crl(grand_name, grand_key, 1).public_bytes(Encoding.DER))
roa_key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
roa_ee = g.sign_cert(x509.Name([g.common_name('Grandchild ROA')]), grand_name, roa_key.public_key(), grand_key,
x509.random_serial_number(), False, roa_key, grand_key,
ip_ext=g.ip_resources([g.CHILD_PREFIX]), signed_object_uri=grand_uri+'valid.roa',
crl_uri=grand_uri+'grand.crl', ca_issuer_uri=child_uri+'grand.cer',
not_before=g.VALID_FROM+timedelta(days=31), not_after=g.VALID_TO-timedelta(days=31))
g.write_cms(g.roa_content(g.CHILD_PREFIX, 24), g.OID_ROA_ECONTENT, roa_ee, roa_key, repository/'grand/valid.roa', work)
manifest('grand', 2, ['grand.crl', 'valid.roa'], grand_name, grand_key, child_uri+'grand.cer')
manifest('child', 5, ['child.crl', 'grand.cer'], child.subject, child_key, child_uri+'child.cer')
manifest('root', 7, ['root.crl', 'child.cer'], root.subject, root_key, root_uri+'ta.cer')
execute('three-level-fresh', 1)
(repository/'root/root.mft').write_bytes(b'bad three-level parent')
execute('three-level-cache', 1, True, no_child_fetch=True)
execute('three-level-cache-ccr-off', 1, True, config_change=lambda s:'\n'.join(l for l in s.splitlines() if not l.startswith('ccr-out ='))+'\n', no_child_fetch=True)
execute('three-level-depth-limit', 0, True, config_change=lambda s:s.replace('require-complete = true', 'require-complete = true\nmax-ca-depth = 1'), no_child_fetch=True)
# A later-expiring sibling stays usable when the first leaf expires.
sibling_key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
sibling_name = x509.Name([g.common_name('Sibling CA')])
sibling = g.sign_cert(sibling_name, child.subject, sibling_key.public_key(), child_key, 910, True, sibling_key, child_key,
ip_ext=g.ip_resources([g.SECOND_PREFIX]), as_ext=g.as_resources([g.TEST_ASN]),
ca_repository=sibling_uri, manifest_uri=sibling_uri+'sibling.mft', notify_uri=notify,
crl_uri=child_uri+'child.crl', ca_issuer_uri=root_uri+'child.cer')
(repository/'child/sibling.cer').write_bytes(sibling.public_bytes(Encoding.DER)); (repository/'sibling').mkdir()
(repository/'sibling/sibling.crl').write_bytes(g.make_crl(sibling_name, sibling_key, 1).public_bytes(Encoding.DER))
ee_key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
ee = g.sign_cert(x509.Name([g.common_name('Sibling ROA')]), sibling_name, ee_key.public_key(), sibling_key,
x509.random_serial_number(), False, ee_key, sibling_key, ip_ext=g.ip_resources([g.SECOND_PREFIX]),
signed_object_uri=sibling_uri+'valid.roa', crl_uri=sibling_uri+'sibling.crl', ca_issuer_uri=child_uri+'sibling.cer',
not_before=g.VALID_FROM+timedelta(days=31), not_after=g.VALID_TO-timedelta(days=10))
g.write_cms(g.roa_content(g.SECOND_PREFIX, 25), g.OID_ROA_ECONTENT, ee, ee_key, repository/'sibling/valid.roa', work)
manifest('sibling', 1, ['sibling.crl', 'valid.roa'], sibling_name, sibling_key, child_uri+'sibling.cer')
manifest('child', 6, ['child.crl', 'grand.cer', 'sibling.cer'], child.subject, child_key, child_uri+'child.cer')
manifest('root', 8, ['root.crl', 'child.cer'], root.subject, root_key, root_uri+'ta.cer')
execute('siblings-fresh', 2)
(repository/'root/root.mft').write_bytes(b'bad sibling parent')
execute('siblings-cache', 2, True, no_child_fetch=True)
execute('one-cached-leaf-expired', 1, True, stamp='2027-12-01T00:00:00Z', no_child_fetch=True)
proof['transports'].append({'transport': transport, 'runs': runs})
finally:
server.shutdown(); server.server_close(); thread.join(timeout=5)
if evidence: evidence.write_text(json.dumps(proof, indent=2)+'\n')
return proof
if __name__ == '__main__':
parser = argparse.ArgumentParser(); parser.add_argument('binary'); parser.add_argument('--evidence', type=Path); args = parser.parse_args()
print(json.dumps(run(args.binary, args.evidence)))