diff --git a/sparse_autoencoder/paths.py b/sparse_autoencoder/paths.py index 9619a11..0759f1c 100644 --- a/sparse_autoencoder/paths.py +++ b/sparse_autoencoder/paths.py @@ -1,3 +1,9 @@ +BASE_URL = "az://openaipublic/sparse-autoencoder/gpt2-small" +VALID_LOCATIONS_V1_V4 = ["mlp_post_act", "resid_delta_mlp"] +VALID_LOCATIONS_V5 = ["resid_delta_attn", "resid_delta_mlp", "resid_post_attn", "resid_post_mlp"] + +def generate_url(location: str, layer_index: int, version: str) -> str: + return f"{BASE_URL}/{location}_{version}/autoencoders/{layer_index}.pt" def v1(location, layer_index): """ @@ -10,18 +16,18 @@ def v1(location, layer_index): - NeuronRecord files: `az://openaipublic/sparse-autoencoder/gpt2-small/{location}/collated_activations/{layer_index}/{latent_index}.json` """ - assert location in ["mlp_post_act", "resid_delta_mlp"] - assert layer_index in range(12) - return f"az://openaipublic/sparse-autoencoder/gpt2-small/{location}/autoencoders/{layer_index}.pt" + assert location in VALID_LOCATIONS_V1_V4, f"Invalid location: {location}" + assert layer_index in range(12), f"Invalid layer_index: {layer_index}" + return f"{BASE_URL}/{location}/autoencoders/{layer_index}.pt" def v4(location, layer_index): """ Details: same as v1 """ - assert location in ["mlp_post_act", "resid_delta_mlp"] - assert layer_index in range(12) - return f"az://openaipublic/sparse-autoencoder/gpt2-small/{location}_v4/autoencoders/{layer_index}.pt" + assert location in VALID_LOCATIONS_V1_V4, f"Invalid location: {location}" + assert layer_index in range(12), f"Invalid layer_index: {layer_index}" + return generate_url(location, layer_index, "v4") def v5_32k(location, layer_index): """ @@ -32,10 +38,10 @@ def v5_32k(location, layer_index): - L1 regularization strength: n/a - Layer normed inputs: true """ - assert location in ["resid_delta_attn", "resid_delta_mlp", "resid_post_attn", "resid_post_mlp"] - assert layer_index in range(12) + assert location in VALID_LOCATIONS_V5, f"Invalid location: {location}" + assert layer_index in range(12), f"Invalid layer_index: {layer_index}" # note: it's actually 2**15 and 2**17 ~= 131k - return f"az://openaipublic/sparse-autoencoder/gpt2-small/{location}_v5_32k/autoencoders/{layer_index}.pt" + return generate_url(location, layer_index, "v5_32k") def v5_128k(location, layer_index): """ @@ -46,10 +52,10 @@ def v5_128k(location, layer_index): - L1 regularization strength: n/a - Layer normed inputs: true """ - assert location in ["resid_delta_attn", "resid_delta_mlp", "resid_post_attn", "resid_post_mlp"] - assert layer_index in range(12) + assert location in VALID_LOCATIONS_V5, f"Invalid location: {location}" + assert layer_index in range(12), f"Invalid layer_index: {layer_index}" # note: it's actually 2**15 and 2**17 ~= 131k - return f"az://openaipublic/sparse-autoencoder/gpt2-small/{location}_v5_128k/autoencoders/{layer_index}.pt" + return generate_url(location, layer_index, "v5_128k") # NOTE: we have larger autoencoders (up to 8M, with varying n and k) trained on layer 8 resid_post_mlp # we may release them in the future