{
	"schema_version": 1,
	"title": "Single-GPU model lifecycle observations",
	"measured_on": "2026-10-16",
	"repository": "https://github.com/superlinked/sie",
	"scope": "Pinned development-source observations, not a released-version, deployment, quality, throughput or reliability guarantee.",
	"artifact_kind": "Sanitized selected measurements, lifecycle observations and reproduction references; original raw bundles are not included.",
	"experiments": [
		{
			"id": "h100_guard_co_residency",
			"claim": "Two guard models remained resident while serving alternating requests; forced admission pressure evicted an unpinned model before the next load.",
			"source": {
				"tested_base_commit": "2e6350e8e3e80d4b52a60c82bac6dae3be74b1e9",
				"tested_tracked_diff_sha256": "a4f5ce501acff327602f4b69abbdd61516a999f2585c844a599bbcf70f41004d",
				"public_runtime_commit": "b342dc55235cef70fe031857e5788e2b0bb3f1be",
				"pull_request": "https://github.com/superlinked/sie/pull/769",
				"qualification": "Named model configurations and serving adapter/loader files match the tested source manifest. The recorded registry snapshot has an intermediate hash; its exact bytes are unavailable in this publication. The retained report identifies its later change as a docstring only; public base-to-final registry changes are docstring-only. Do not treat the public commit as a byte-identical full snapshot."
			},
			"hardware": {
				"gpu": "NVIDIA H100 80GB HBM3",
				"gpu_count": 1,
				"reported_device_total_bytes": 85009367040
			},
			"software": {
				"torch": "2.9.1+cu129",
				"transformers": "5.3.0",
				"sglang": "0.5.10.post1"
			},
			"models": [
				{
					"model": "Qwen/Qwen3Guard-Gen-4B",
					"revision": "6ec42827da0c1ff11e7a49dc269d2e810d27e108",
					"kv_token_budget": 32768,
					"config_url": "https://huggingface.co/Qwen/Qwen3Guard-Gen-4B/blob/6ec42827da0c1ff11e7a49dc269d2e810d27e108/config.json"
				},
				{
					"model": "ibm-granite/granite-guardian-3.0-2b",
					"revision": "e48b7b8acf438d24daa2271ada6df945b5b8895e",
					"kv_token_budget": 8192,
					"config_url": "https://huggingface.co/ibm-granite/granite-guardian-3.0-2b/blob/e48b7b8acf438d24daa2271ada6df945b5b8895e/config.json"
				}
			],
			"protocol": {
				"recorded_completed_runs": 1,
				"co_residency_rounds": 3,
				"requests_per_round": 2,
				"max_generated_tokens": 32,
				"temperature": 0.0,
				"transport": "SIEClient chat_completions through local SIE HTTP",
				"memory_method": "Device total minus cuda.mem_get_info free bytes after a response; not per-model allocations.",
				"pressure_method": "Unload Granite; allocate a uint8 CUDA ballast tensor to put free memory below its admission reservation; request Granite load."
			},
			"measurements": {
				"successful_guard_requests": 6,
				"co_resident_models": 2,
				"each_engine_pid_unchanged": true,
				"device_free_bytes_with_both_models": 60559130624,
				"device_used_bytes_with_both_models": 24450236416,
				"device_used_gib_with_both_models": 22.77105712890625,
				"sum_admission_reservations_bytes": 26647225703,
				"kv_token_capacities": {
					"Qwen/Qwen3Guard-Gen-4B": 32768,
					"ibm-granite/granite-guardian-3.0-2b": 8192
				},
				"forced_pressure_free_bytes": 8984199168,
				"next_load_reservation_bytes": 9254673613,
				"preload_eviction_passed": true,
				"explicit_memory_fraction_0_85_preserved": true
			},
			"selected_samples": [
				{
					"model": "Qwen/Qwen3Guard-Gen-4B",
					"round": 0,
					"resident_model_count": 1,
					"device_free_bytes": 68537090048,
					"admission_reservations_bytes": {
						"Qwen/Qwen3Guard-Gen-4B": 17392552090
					}
				},
				{
					"model": "ibm-granite/granite-guardian-3.0-2b",
					"round": 0,
					"resident_model_count": 2,
					"device_free_bytes": 60559130624,
					"admission_reservations_bytes": {
						"Qwen/Qwen3Guard-Gen-4B": 17392552090,
						"ibm-granite/granite-guardian-3.0-2b": 9254673613
					}
				},
				{
					"model": "Qwen/Qwen3Guard-Gen-4B",
					"round": 1,
					"resident_model_count": 2,
					"device_free_bytes": 60559130624,
					"admission_reservations_bytes": {
						"Qwen/Qwen3Guard-Gen-4B": 17392552090,
						"ibm-granite/granite-guardian-3.0-2b": 9254673613
					}
				},
				{
					"model": "ibm-granite/granite-guardian-3.0-2b",
					"round": 1,
					"resident_model_count": 2,
					"device_free_bytes": 60559130624,
					"admission_reservations_bytes": {
						"Qwen/Qwen3Guard-Gen-4B": 17392552090,
						"ibm-granite/granite-guardian-3.0-2b": 9254673613
					}
				},
				{
					"model": "Qwen/Qwen3Guard-Gen-4B",
					"round": 2,
					"resident_model_count": 2,
					"device_free_bytes": 60559130624,
					"admission_reservations_bytes": {
						"Qwen/Qwen3Guard-Gen-4B": 17392552090,
						"ibm-granite/granite-guardian-3.0-2b": 9254673613
					}
				},
				{
					"model": "ibm-granite/granite-guardian-3.0-2b",
					"round": 2,
					"resident_model_count": 2,
					"device_free_bytes": 60559130624,
					"admission_reservations_bytes": {
						"Qwen/Qwen3Guard-Gen-4B": 17392552090,
						"ibm-granite/granite-guardian-3.0-2b": 9254673613
					}
				}
			],
			"reproduction": {
				"status": "Methods and data, not a complete runnable reproduction: the exact acquisition script was not committed.",
				"model_configuration_urls": [
					"https://github.com/superlinked/sie/blob/b342dc55235cef70fe031857e5788e2b0bb3f1be/packages/sie_server/models/Qwen__Qwen3Guard-Gen-4B.yaml",
					"https://github.com/superlinked/sie/blob/b342dc55235cef70fe031857e5788e2b0bb3f1be/packages/sie_server/models/ibm-granite__granite-guardian-3.0-2b.yaml"
				],
				"retained_acquisition_script_sha256": "1e17821730f8ca75d722636bc8854c4c6d4ccf92905b324172cd0730925c7e4f"
			},
			"retained_evidence_sha256": {
				"report": "ca404b36ff68d5717e0f932ce179e81fa3448f9d2876497634dc87acff1eda51",
				"raw_events": "b366591ad9eb5e005d4657c68d341eb8db73b240230bf14a662bde5c43b14888",
				"source_manifest_file": "a9f9d00bd105425df0439aab32593096234cef7fa946ae6859bf20d0384e9c45"
			},
			"limitations": [
				"One successful protocol run, two checkpoints, one H100; no replication or confidence interval.",
				"Forced low-free-memory admission does not test CUDA OOM recovery.",
				"Admission reservations and device usage are different quantities.",
				"No throughput, cross-model interference or hardware isolation measurement."
			]
		},
		{
			"id": "h100_single_process_crash",
			"claim": "The healthy guard completed all sampled requests while one neighboring engine was killed and reloaded.",
			"source": {
				"serving_commit": "b342dc55235cef70fe031857e5788e2b0bb3f1be",
				"exact_measured_harness_commit": "7b4d7d9b0039d8a1c206256eff4aad35ceaa5c93",
				"pull_request": "https://github.com/superlinked/sie/pull/773"
			},
			"hardware": {
				"gpu": "NVIDIA H100 80GB HBM3",
				"gpu_count": 1,
				"reported_device_total_bytes": 85009367040,
				"cpu_cores": 8,
				"host_memory_gib": 64
			},
			"software": {
				"sglang": "0.5.10.post1",
				"torch": "2.9.1+cu129",
				"transformers": "5.3.0"
			},
			"models": [
				{
					"model": "Qwen/Qwen3Guard-Gen-4B",
					"revision": "6ec42827da0c1ff11e7a49dc269d2e810d27e108",
					"kv_token_budget": 32768,
					"config_url": "https://huggingface.co/Qwen/Qwen3Guard-Gen-4B/blob/6ec42827da0c1ff11e7a49dc269d2e810d27e108/config.json"
				},
				{
					"model": "ibm-granite/granite-guardian-3.0-2b",
					"revision": "e48b7b8acf438d24daa2271ada6df945b5b8895e",
					"kv_token_budget": 8192,
					"config_url": "https://huggingface.co/ibm-granite/granite-guardian-3.0-2b/blob/e48b7b8acf438d24daa2271ada6df945b5b8895e/config.json"
				}
			],
			"protocol": {
				"recorded_completed_runs": 1,
				"crashes": 1,
				"fault": "SIGKILL to the owned Granite engine child",
				"pinned_healthy_model": "Qwen/Qwen3Guard-Gen-4B",
				"victim_model": "ibm-granite/granite-guardian-3.0-2b",
				"pause_between_healthy_requests_s": 2,
				"requested_before_samples": 25,
				"during_phase_sample_limit": 50,
				"requested_after_samples": 25,
				"transport": "SIEClient through local SIE HTTP",
				"cooldown_s": 30,
				"harness_deadline_s": 1300,
				"recovery_timeout_s": 300
			},
			"measurements": {
				"healthy_requests_total": 96,
				"healthy_requests_successful": 96,
				"phase_counts": {
					"before": 25,
					"during": 46,
					"after": 25
				},
				"healthy_engine_pid_unchanged": true,
				"victim_engine_pid_changed": true,
				"successful_reloads": 1,
				"detection_after_crash_s": 1.0768107920000034,
				"recovery_complete_marker_after_crash_s": 91.16530184000001,
				"during_phase_window_relative_to_crash_s": {
					"first_request": 0.06663327199999003,
					"last_response": 92.87876951400003
				}
			},
			"selected_lifecycle_events": [
				{
					"event": "crash",
					"seconds_after_crash": 0.0,
					"response_statuses": []
				},
				{
					"event": "crash_detected",
					"seconds_after_crash": 1.0768107920000034,
					"response_statuses": []
				},
				{
					"event": "recovery_request",
					"seconds_after_crash": 1.0807442130000027,
					"response_statuses": [502]
				},
				{
					"event": "recovery_request",
					"seconds_after_crash": 61.111804422000006,
					"response_statuses": [503, 503, 503, 503, 503, 503, 503]
				},
				{
					"event": "recovery_request",
					"seconds_after_crash": 91.16401009600006,
					"response_statuses": [503, 503, 503, 503, 503, 200]
				},
				{
					"event": "recovered",
					"seconds_after_crash": 91.16530184000001,
					"response_statuses": []
				}
			],
			"caller_behavior": [
				"During ENGINE cooldown: 502 MODEL_LOAD_FAILED, permanent=false, no Retry-After; SDK does not auto-retry this error.",
				"After observed 30-second cooldown the harness sends a fresh request. Loading returns 503 with Retry-After: 5.",
				"First reload request reaches the harness 30-second provisioning limit. A subsequent request joins the same ongoing reload and succeeds; this is one new engine process, not a second reload."
			],
			"reproduction": {
				"readme_url": "https://github.com/superlinked/sie/blob/7b4d7d9b0039d8a1c206256eff4aad35ceaa5c93/tools/benchmarks/model_recovery/README.md",
				"probe_url": "https://github.com/superlinked/sie/blob/7b4d7d9b0039d8a1c206256eff4aad35ceaa5c93/tools/benchmarks/model_recovery/probe.py",
				"invocation": [
					"python",
					"tools/benchmarks/model_recovery/probe.py",
					"--output",
					"<fresh-output-directory>"
				],
				"environment_note": "Use the recorded SGLang bundle versions on an isolated H100; the README documents CPU smoke, prerequisites and bounded cleanup."
			},
			"retained_evidence_sha256": {
				"report": "441554e27815e8d26b9f4f5e945e921d64bcc51fe798bcfd497f7f8784c69ed5",
				"raw_events": "a3d1a717b9fcd5983813457f8eaa65aefbcead611944a98d328f0844f26bb7ba",
				"raw_bundle": "5842df213e447633a6779180e0186d4e2ffc0b9d6f1ebb8961e37c37b36e1af3"
			},
			"limitations": [
				"One crash at low request rate; 96 successes are an observation, not an availability guarantee.",
				"The recovery sampling phase covers the observed recovery and ends slightly after it.",
				"Cooldown errors require a fresh request; recovery was neither transparent nor immediate.",
				"No CUDA OOM, repeated-crash, retry-exhaustion, high-load fairness, hardware-isolation or throughput result.",
				"Pinning was configured; its causal benefit against eviction was not isolated."
			]
		},
		{
			"id": "l4_runtime_lora_lifecycle",
			"claim": "Declared generation and embedding adapters changed output and could be evicted and reloaded without restarting their engine; warm base output was restored.",
			"source": {
				"tested_commit": "924305db79246142bd15355a479a994536f968bb",
				"pull_request": "https://github.com/superlinked/sie/pull/774",
				"source_content_digest_sha256": "792d9a559750b6ef2c6dccdad5753a8eaee95c343bf13e63336592baa866beb1",
				"source_manifest_file_sha256": "f3adf6471ebd6ab723212abf10d3dcd319eac632c1ffc8e1d5fd0e5ea1289d52",
				"verified_source_files": 334,
				"source_content_digest_definition": "SHA-256 of json.dumps(file_path_to_sha256, sort_keys=True).encode(); distinct from the manifest file byte hash."
			},
			"hardware": {
				"gpu": "NVIDIA L4",
				"gpu_count": 1,
				"reported_device_total_bytes": 23689363456,
				"cpu_cores": 8,
				"host_memory_gib": 32,
				"model_execution": "Generation and embedding engines run sequentially."
			},
			"software": {
				"torch": "2.9.1+cu129",
				"transformers": "5.3.0",
				"sglang": "0.5.10.post1",
				"peft": "0.20.0"
			},
			"models": [
				{
					"kind": "generation",
					"model": "Qwen/Qwen3-0.6B",
					"revision": "c1899de289a04d12100db370d81485cdf75e47ca",
					"config_sha256": "660db3b73d788119c04535e48cf9be5f55bc3100841a718637ae695b442f27dd",
					"config_url": "https://huggingface.co/Qwen/Qwen3-0.6B/blob/c1899de289a04d12100db370d81485cdf75e47ca/config.json"
				},
				{
					"kind": "embedding",
					"model": "Qwen/Qwen3-Embedding-0.6B",
					"revision": "97b0c614be4d77ee51c0cef4e5f07c00f9eb65b3",
					"config_sha256": "b5bf1f51fc45be473a54718cef92448d90a1be001bf9b9a44b8c7f10a19feaa9",
					"config_url": "https://huggingface.co/Qwen/Qwen3-Embedding-0.6B/blob/97b0c614be4d77ee51c0cef4e5f07c00f9eb65b3/config.json"
				}
			],
			"protocol": {
				"recorded_completed_runs": 1,
				"engine_runs": {
					"generation": 1,
					"embedding": 1
				},
				"fixtures": "Deterministic untrained rank-4 A/B weights on q_proj and v_proj; no trained task adapter.",
				"fixture_seeds": {
					"pin": 20261016,
					"a": 20261017,
					"b": 20261018,
					"c": 20261019,
					"d": 20261020
				},
				"lora_alpha": 8,
				"fixture_normal_weight_scale": 0.04,
				"static_memory_fraction": 0.5,
				"kv_token_cap": 4096,
				"ready_adapter_capacity": 3,
				"preloaded_pinned_aliases": ["pin"],
				"dynamic_aliases": ["a", "b", "c", "d"],
				"actual_generation_gpu_slots": 2,
				"churn_request_order": ["c", "d", "a", "b", "c", "d", "pin"],
				"temperature": 0.0,
				"request_seed": 17,
				"preselection_controls": "First response and two warm baseline repeats complete before any runtime adapter selection.",
				"base_comparison": "Restored output versus second warm baseline: native generation text identical, observed aligned probability drift < 1e-3, raw embedding drift < 5e-5.",
				"effect_thresholds": {
					"generation_first_token_probability_lower_bound": 0.0001,
					"embedding_max_abs": 1e-5,
					"effect_vs_control_drift_multiplier": 5
				},
				"embedding_raw_vector_shape": [2, 1024],
				"instrumentation": "Read-only generation scheduler observer records successful forwards and distinct GPU slots. A benchmark embedding subclass adds a fixed 4096-token KV cap while preserving normal launch and inference paths."
			},
			"measurements": {
				"successful_forwards_containing_both_a_and_b": 22,
				"mixed_forward_count_is_not_request_or_trial_count": true,
				"distinct_valid_gpu_slots_for_a_and_b": true,
				"engine_pid_stable_for_each_model": true,
				"ready_listing_capacity_and_pin_preserved": true,
				"dynamic_aliases_evicted_and_served_again": ["a", "b", "c", "d"],
				"cold_loading_503_then_success_aliases": ["a", "b", "c", "d"],
				"undeclared_adapter_status": 400,
				"warm_base_generation_text_restored": true,
				"observed_warm_generation_probability_delta": 0.0,
				"embedding_a_vs_b_max_abs": 0.10613857954740524,
				"embedding_a_vs_warm_base_max_abs": 0.06803027167916298,
				"embedding_b_vs_warm_base_max_abs": 0.08349677361547947,
				"embedding_warm_repeat_max_abs": 0.0,
				"embedding_reset_after_a_b_max_abs": 0.0,
				"embedding_reset_after_churn_max_abs": 0.0
			},
			"selected_residency_observations": [
				{
					"kind": "generation",
					"phase": "initial",
					"ready_aliases": ["pin"]
				},
				{
					"kind": "generation",
					"phase": "a_b_ready",
					"ready_aliases": ["a", "b", "pin"]
				},
				{
					"kind": "generation",
					"phase": "before_churn",
					"ready_aliases": ["a", "b", "pin"]
				},
				{
					"kind": "generation",
					"phase": "churn_0",
					"ready_aliases": ["a", "c", "pin"]
				},
				{
					"kind": "generation",
					"phase": "churn_1",
					"ready_aliases": ["c", "d", "pin"]
				},
				{
					"kind": "generation",
					"phase": "churn_2",
					"ready_aliases": ["a", "d", "pin"]
				},
				{
					"kind": "generation",
					"phase": "churn_3",
					"ready_aliases": ["a", "b", "pin"]
				},
				{
					"kind": "generation",
					"phase": "churn_4",
					"ready_aliases": ["b", "c", "pin"]
				},
				{
					"kind": "generation",
					"phase": "churn_5",
					"ready_aliases": ["c", "d", "pin"]
				},
				{
					"kind": "generation",
					"phase": "churn_6",
					"ready_aliases": ["c", "d", "pin"]
				},
				{
					"kind": "embedding",
					"phase": "initial",
					"ready_aliases": ["pin"]
				},
				{
					"kind": "embedding",
					"phase": "a_b_ready",
					"ready_aliases": ["a", "b", "pin"]
				},
				{
					"kind": "embedding",
					"phase": "before_churn",
					"ready_aliases": ["a", "b", "pin"]
				},
				{
					"kind": "embedding",
					"phase": "churn_0",
					"ready_aliases": ["b", "c", "pin"]
				},
				{
					"kind": "embedding",
					"phase": "churn_1",
					"ready_aliases": ["c", "d", "pin"]
				},
				{
					"kind": "embedding",
					"phase": "churn_2",
					"ready_aliases": ["a", "d", "pin"]
				},
				{
					"kind": "embedding",
					"phase": "churn_3",
					"ready_aliases": ["a", "b", "pin"]
				},
				{
					"kind": "embedding",
					"phase": "churn_4",
					"ready_aliases": ["b", "c", "pin"]
				},
				{
					"kind": "embedding",
					"phase": "churn_5",
					"ready_aliases": ["c", "d", "pin"]
				},
				{
					"kind": "embedding",
					"phase": "churn_6",
					"ready_aliases": ["c", "d", "pin"]
				}
			],
			"controls": {
				"cold_vs_warm_chat_max_observed_chosen_probability_delta": 0.058946889506815925,
				"cold_vs_warm_native_nonstreaming_text_equal": false,
				"cold_vs_warm_embedding_max_abs": 0.0025426670908927917,
				"warm_repeats_and_resets_match": true,
				"prior_negative_result": "Earlier cold-to-reset comparisons failed. The final run reproduces cold-to-warm differences before runtime selection while warm repeats and resets match under unchanged thresholds. The numerical cause was not established."
			},
			"caller_behavior": [
				"Encode SDK retries LORA_LOADING automatically.",
				"Generation harness makes bounded fresh SDK calls only for pre-execution 503 LORA_LOADING; this is not automatic generation SDK retry.",
				"Opened failed streams are not retried."
			],
			"reproduction": {
				"readme_url": "https://github.com/superlinked/sie/blob/924305db79246142bd15355a479a994536f968bb/tools/benchmarks/runtime_lora/README.md",
				"fixture_generator_url": "https://github.com/superlinked/sie/blob/924305db79246142bd15355a479a994536f968bb/tools/benchmarks/runtime_lora/fixtures.py",
				"fixture_invocation": [
					"python",
					"-m",
					"tools.benchmarks.runtime_lora.fixtures",
					"--output",
					"<fixture-directory>",
					"--download-weights"
				],
				"probe_invocation": [
					"python",
					"-m",
					"tools.benchmarks.runtime_lora.probe",
					"--fixtures",
					"<fixture-directory>",
					"--output",
					"<fresh-output-directory>"
				],
				"environment_note": "Use the recorded compatible CUDA/SGLang environment. Fixtures depend only on public pinned models and harness source. The probe has a 780-second default deadline."
			},
			"retained_evidence_sha256": {
				"summary": "825127b949633e87e9065110f3086d3608ed9af056583bc2c5e5b4f166d6092a",
				"verdict": "f871a2a655210da479c625b919f35aff0eb96485645431d3ac5397c03ad30475",
				"raw_events": "073cfb6f50ffa81311b2fe75a0a9a42115821d9e411c028eae2e4ebf0f8600b9",
				"raw_batches": "0db3c875b77587d953d108d94c7cc2dd85f28e685c08cbcad52e35467a8cbc5e",
				"fixture_manifest": "8fa978d4c9a72f6c3293c4c791fc48dcd23a57cbc4803c9cd27534fccc27c18e",
				"raw_bundle": "8cc8bb65d33a8080c12ee0a007bb2b8cfb95b798434323dcf9342936a046698b"
			},
			"limitations": [
				"One final successful run on one L4; earlier negative comparisons remain part of the evidence.",
				"Untrained fixtures establish selection and restoration, not model quality or customer demand.",
				"Generation batching is observed; embedding selection remains serialized and no mixed-adapter embedding forward is claimed.",
				"Observed top-k token probabilities do not bound the unreported distribution tail.",
				"Warm restoration does not guarantee cold and warm outputs are identical.",
				"No throughput, other-adapter-family or other-SGLang-version acceptance claim."
			]
		}
	],
	"integrity_note": "These SHA-256 values identify retained originals. This record publishes selected measurements and public harness links, not the original raw bundles."
}