diff --git a/applications/jaxrs/pom.xml b/applications/jaxrs/pom.xml new file mode 100644 index 000000000..6a9de5519 --- /dev/null +++ b/applications/jaxrs/pom.xml @@ -0,0 +1,138 @@ + + + + + + 4.0.0 + war + + com.google.appengine + applications + 4.0.1-SNAPSHOT + + com.google.appengine.demos + jaxrs + AppEngine :: jaxrs + https://github.com/GoogleCloudPlatform/appengine-java-standard/ + A jaxrs sample application. + + + 3.6.0 + + + + true + 4.0.1-SNAPSHOT + UTF-8 + 1.8 + 1.8 + + + + + + javax.servlet + javax.servlet-api + provided + + + org.glassfish.jersey.containers + jersey-container-servlet + 2.47 + + + + org.glassfish.jersey.inject + jersey-hk2 + 2.47 + + + + org.glassfish.jersey.connectors + jersey-jetty-connector + 2.47 + + + + + target/${project.artifactId}-${project.version}/WEB-INF/classes + + + + org.eclipse.jetty.ee8 + jetty-ee8-maven-plugin + 12.1.5 + + 10 + true + + /test + + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.5.4 + + + --add-opens java.base/java.lang=ALL-UNNAMED + --add-opens java.base/java.nio.charset=ALL-UNNAMED + --add-opens java.base/java.util=ALL-UNNAMED + --add-opens java.base/java.util.concurrent=ALL-UNNAMED + + + + + org.apache.maven.plugins + maven-war-plugin + 3.5.1 + + true + + + + ${basedir}/src/main/webapp/WEB-INF + true + WEB-INF + + + + + + + com.google.cloud.tools + appengine-maven-plugin + 2.8.6 + + ludo-in-in + jaxrs + false + true + + -Xdebug + -agentlib:jdwp=transport=dt_socket,address=8000,server=y,suspend=n + + 553.0.0 + + + + + + diff --git a/applications/jaxrs/src/main/java/org/example/AppConfig.java b/applications/jaxrs/src/main/java/org/example/AppConfig.java new file mode 100644 index 000000000..66fbde4a8 --- /dev/null +++ b/applications/jaxrs/src/main/java/org/example/AppConfig.java @@ -0,0 +1,26 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.example; + +import javax.ws.rs.ApplicationPath; +import org.glassfish.jersey.server.ResourceConfig; + +@ApplicationPath("/") +public class AppConfig extends ResourceConfig { + public AppConfig() { + register(RootResource.class); + } +} diff --git a/applications/jaxrs/src/main/java/org/example/HelloResource.java b/applications/jaxrs/src/main/java/org/example/HelloResource.java new file mode 100644 index 000000000..8a4d3d68c --- /dev/null +++ b/applications/jaxrs/src/main/java/org/example/HelloResource.java @@ -0,0 +1,28 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.example; + +import javax.ws.rs.GET; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; + +public class HelloResource { + @GET + @Produces(MediaType.TEXT_PLAIN) + public String hello() { + return "hello"; + } +} diff --git a/applications/jaxrs/src/main/java/org/example/RootResource.java b/applications/jaxrs/src/main/java/org/example/RootResource.java new file mode 100644 index 000000000..c098ba4ae --- /dev/null +++ b/applications/jaxrs/src/main/java/org/example/RootResource.java @@ -0,0 +1,26 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.example; + +import javax.ws.rs.Path; + +@Path("/") +public class RootResource { + @Path("hello") + public HelloResource getHelloResource() { + return new HelloResource(); + } +} diff --git a/applications/jaxrs/src/main/webapp/WEB-INF/appengine-web.xml b/applications/jaxrs/src/main/webapp/WEB-INF/appengine-web.xml new file mode 100644 index 000000000..25df2b6fa --- /dev/null +++ b/applications/jaxrs/src/main/webapp/WEB-INF/appengine-web.xml @@ -0,0 +1,24 @@ + + + + + java17 + + + + + diff --git a/applications/jaxrs/src/main/webapp/WEB-INF/logging.properties b/applications/jaxrs/src/main/webapp/WEB-INF/logging.properties new file mode 100644 index 000000000..fe435d2c1 --- /dev/null +++ b/applications/jaxrs/src/main/webapp/WEB-INF/logging.properties @@ -0,0 +1,27 @@ +# +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# A default java.util.logging configuration. +# (All App Engine logging is through java.util.logging by default). +# +# To use this configuration, copy it into your application's WEB-INF +# folder and add the following to your appengine-web.xml: +# +# +# +# + +# Set the default logging level for all loggers to WARNING +.level = WARNING diff --git a/applications/pom.xml b/applications/pom.xml index dd1c057d2..56f39239d 100644 --- a/applications/pom.xml +++ b/applications/pom.xml @@ -38,5 +38,6 @@ guestbook_jakarta servletasyncapp servletasyncappjakarta + jaxrs diff --git a/e2etests/devappservertests/src/test/java/com/google/appengine/tools/development/JaxRsTest.java b/e2etests/devappservertests/src/test/java/com/google/appengine/tools/development/JaxRsTest.java new file mode 100644 index 000000000..796d735da --- /dev/null +++ b/e2etests/devappservertests/src/test/java/com/google/appengine/tools/development/JaxRsTest.java @@ -0,0 +1,70 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.appengine.tools.development; + + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class JaxRsTest extends DevAppServerTestBase { + + @Parameterized.Parameters + public static List version() { + // Only EE8 app. +return Arrays.asList( + new Object[][] { + {"java17", "9.4", "EE6"}, + {"java17", "12.0", "EE8"}, + {"java21", "12.0", "EE8"}, + {"java25", "12.1", "EE8"}, + + }); + + } + public JaxRsTest(String runtimeVersion, String jettyVersion, String jakartaVersion) { + super(runtimeVersion, jettyVersion, jakartaVersion); + } + + @Before + public void setUpClass() throws IOException, InterruptedException { + File currentDirectory = new File("").getAbsoluteFile(); + File appRoot = + new File( + currentDirectory, + "../../applications/jaxrs/target/jaxrs" + + "-" + + System.getProperty("appengine.projectversion")); + setUpClass(appRoot); + } + + @Test + public void testJaxRs() throws Exception { + // App Engine Memcache access. + executeHttpGet( + "/hello", + "hello", + RESPONSE_200); + + } + +} diff --git a/remoteapi/pom.xml b/remoteapi/pom.xml index dcca2df13..7dd4f5fcd 100644 --- a/remoteapi/pom.xml +++ b/remoteapi/pom.xml @@ -56,6 +56,37 @@ com.google.appengine appengine-api-1.0-sdk + + junit + junit + test + + + com.google.truth + truth + test + + + com.google.truth.extensions + truth-proto-extension + 1.4.5 + test + + + org.mockito + mockito-core + test + + + com.google.protobuf + protobuf-java + true + + + org.apache.httpcomponents + httpclient + true + diff --git a/remoteapi/src/main/java/com/google/appengine/tools/remoteapi/RemoteDatastore.java b/remoteapi/src/main/java/com/google/appengine/tools/remoteapi/RemoteDatastore.java index c6fcceeea..a5dd353f3 100644 --- a/remoteapi/src/main/java/com/google/appengine/tools/remoteapi/RemoteDatastore.java +++ b/remoteapi/src/main/java/com/google/appengine/tools/remoteapi/RemoteDatastore.java @@ -23,7 +23,6 @@ import com.google.protobuf.Message; import com.google.storage.onestore.v3_bytes.proto2api.OnestoreEntity; import java.util.ArrayList; -import java.util.Collection; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -195,12 +194,11 @@ static boolean rewriteQueryAppIds(DatastoreV3Pb.Query.Builder query, String remo reserialize = true; query.getAncestorBuilder().setApp(remoteAppId); } - for (DatastoreV3Pb.Query.Filter filter : query.getFilterList()) { - for (OnestoreEntity.Property prop : filter.getPropertyList()) { - OnestoreEntity.PropertyValue propValue = prop.getValue(); - if (propValue.hasReferenceValue()) { + for (DatastoreV3Pb.Query.Filter.Builder filter : query.getFilterBuilderList()) { + for (OnestoreEntity.Property.Builder prop : filter.getPropertyBuilderList()) { + if (prop.getValue().hasReferenceValue()) { OnestoreEntity.PropertyValue.ReferenceValue.Builder ref = - propValue.getReferenceValue().toBuilder(); + prop.getValueBuilder().getReferenceValueBuilder(); if (!ref.getApp().equals(remoteAppId)) { reserialize = true; ref.setApp(remoteAppId); @@ -268,7 +266,7 @@ private byte[] handleGet(byte[] originalRequestBytes) { mergeFromBytes(rewrittenReq, originalRequestBytes); // Update the Request so that all References have the remoteAppId. - boolean reserialize = rewriteRequestReferences(rewrittenReq.getKeyList(), remoteAppId); + boolean reserialize = rewriteRequestReferences(rewrittenReq.getKeyBuilderList(), remoteAppId); if (rewrittenReq.hasTransaction()) { return handleGetWithTransaction(rewrittenReq.build()); } else { @@ -326,7 +324,7 @@ private byte[] handleDelete(byte[] requestBytes) { DatastoreV3Pb.DeleteRequest.Builder request = DatastoreV3Pb.DeleteRequest.newBuilder(); mergeFromBytes(request, requestBytes); - boolean reserialize = rewriteRequestReferences(request.getKeyList(), remoteAppId); + boolean reserialize = rewriteRequestReferences(request.getKeyBuilderList(), remoteAppId); if (reserialize) { // The request was mutated, so we need to reserialize it. requestBytes = request.build().toByteArray(); @@ -348,12 +346,12 @@ private byte[] handleDelete(byte[] requestBytes) { */ /* @VisibleForTesting */ static boolean rewriteRequestReferences( - Collection references, String remoteAppId) { + List references, String remoteAppId) { boolean reserialize = false; - for (OnestoreEntity.Reference refToCheck : references) { + for (OnestoreEntity.Reference.Builder refToCheck : references) { if (!refToCheck.getApp().equals(remoteAppId)) { - refToCheck = refToCheck.toBuilder().setApp(remoteAppId).build(); + refToCheck.setApp(remoteAppId); reserialize = true; } } diff --git a/remoteapi/src/main/java/com/google/appengine/tools/remoteapi/RemoteRpc.java b/remoteapi/src/main/java/com/google/appengine/tools/remoteapi/RemoteRpc.java index 162b86ccb..4544885d7 100644 --- a/remoteapi/src/main/java/com/google/appengine/tools/remoteapi/RemoteRpc.java +++ b/remoteapi/src/main/java/com/google/appengine/tools/remoteapi/RemoteRpc.java @@ -1,4 +1,3 @@ - /* * Copyright 2021 Google LLC * diff --git a/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/OAuthClientTest.java b/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/OAuthClientTest.java new file mode 100644 index 000000000..05246be3b --- /dev/null +++ b/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/OAuthClientTest.java @@ -0,0 +1,122 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.tools.remoteapi; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import com.google.api.client.http.LowLevelHttpRequest; +import com.google.api.client.http.LowLevelHttpResponse; +import com.google.api.client.testing.http.MockHttpTransport; +import com.google.api.client.testing.http.MockLowLevelHttpRequest; +import com.google.api.client.testing.http.MockLowLevelHttpResponse; +import com.google.appengine.tools.remoteapi.AppEngineClient.Response; +import com.google.appengine.tools.remoteapi.testing.StubCredential; +import java.io.IOException; +import java.util.Arrays; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Test for {@link OAuthClient}. + */ +@RunWith(JUnit4.class) +public class OAuthClientTest { + + private static final String APP_ID = "appid"; + + private static class SimpleHttpTransport extends MockHttpTransport { + private final byte[] responseBytes; + + private String lastMethod = null; + private String lastUrl = null; + + SimpleHttpTransport(byte[] responseBytes) { + this.responseBytes = responseBytes; + } + + @Override + public LowLevelHttpRequest buildRequest(String method, String url) throws IOException { + lastMethod = method; + lastUrl = url; + return new MockLowLevelHttpRequest() { + @Override + public LowLevelHttpResponse execute() throws IOException { + MockLowLevelHttpResponse response = new MockLowLevelHttpResponse(); + response.setContent(Arrays.copyOf(responseBytes, responseBytes.length)); + return response; + } + }; + } + + String getLastMethod() { + return lastMethod; + } + + String getLastUrl() { + return lastUrl; + } + } + + @Test + public void testConstructor() { + RemoteApiOptions noOAuthCredential = new RemoteApiOptions() + .credentials("email", "password") + .httpTransport(new SimpleHttpTransport(new byte[] {})); + assertThrows(IllegalArgumentException.class, () -> new OAuthClient(noOAuthCredential, APP_ID)); + + RemoteApiOptions noHttpTransport = new RemoteApiOptions() + .oauthCredential(new StubCredential()); + assertThrows(IllegalStateException.class, () -> new OAuthClient(noHttpTransport, APP_ID)); + } + + @Test + public void testGet() throws Exception { + byte[] expectedResponseBytes = new byte[] {42}; + + SimpleHttpTransport transport = new SimpleHttpTransport(expectedResponseBytes); + RemoteApiOptions options = new RemoteApiOptions() + .server("example.com", 8080) + .oauthCredential(new StubCredential()) + .httpTransport(transport); + + Response response = new OAuthClient(options, APP_ID) + .get("/foo"); + assertTrue(Arrays.equals(expectedResponseBytes, response.getBodyAsBytes())); + assertEquals("GET", transport.getLastMethod()); + assertEquals("http://example.com:8080/foo", transport.getLastUrl()); + } + + @Test + public void testPost() throws Exception { + byte[] expectedResponseBytes = new byte[] {42}; + + SimpleHttpTransport transport = new SimpleHttpTransport(expectedResponseBytes); + RemoteApiOptions options = new RemoteApiOptions() + .server("example.com", 443) + .oauthCredential(new StubCredential()) + .httpTransport(transport); + + Response response = new OAuthClient(options, APP_ID) + .post("/foo", "application/json", new byte[]{1}); + assertTrue(Arrays.equals(expectedResponseBytes, response.getBodyAsBytes())); + assertEquals("POST", transport.getLastMethod()); + assertEquals("https://example.com:443/foo", transport.getLastUrl()); + } +} diff --git a/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/RemoteApiDelegateTest.java b/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/RemoteApiDelegateTest.java new file mode 100644 index 000000000..b70841437 --- /dev/null +++ b/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/RemoteApiDelegateTest.java @@ -0,0 +1,840 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.tools.remoteapi; + +import static com.google.common.io.BaseEncoding.base64Url; +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth.assertWithMessage; +import static com.google.common.truth.extensions.proto.ProtoTruth.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import com.google.appengine.api.datastore.DatastoreService; +import com.google.appengine.api.datastore.DatastoreServiceFactory; +import com.google.appengine.api.datastore.Entity; +import com.google.appengine.api.datastore.EntityNotFoundException; +import com.google.appengine.api.datastore.EntityTranslator; +import com.google.appengine.api.datastore.FetchOptions; +import com.google.appengine.api.datastore.Key; +import com.google.appengine.api.datastore.KeyFactory; +import com.google.appengine.api.datastore.PreparedQuery; +import com.google.appengine.api.datastore.Query; +import com.google.appengine.api.datastore.Transaction; +import com.google.appengine.api.memcache.MemcacheServiceFactory; +import com.google.appengine.api.memcache.MemcacheServicePb; +import com.google.apphosting.api.ApiProxy; +import com.google.apphosting.base.protos.api_bytes.RemoteApiPb; +import com.google.apphosting.datastore_bytes.proto2api.DatastoreV3Pb; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.google.common.hash.Hashing; +import com.google.errorprone.annotations.CanIgnoreReturnValue; +import com.google.protobuf.ByteString; +import com.google.protobuf.ExtensionRegistry; +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.Message; +import com.google.storage.onestore.v3_bytes.proto2api.OnestoreEntity; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.ObjectOutput; +import java.io.ObjectOutputStream; +import java.util.AbstractMap.SimpleImmutableEntry; +import java.util.Iterator; +import java.util.List; +import java.util.Queue; +import java.util.concurrent.LinkedBlockingQueue; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Verifies that the remote API makes the right RPC calls. + * + */ +@RunWith(JUnit4.class) +public class RemoteApiDelegateTest { + private MockRpc rpc; + private DatastoreService datastore; + private RemoteApiInstaller installer; + + @Before + @SuppressWarnings("deprecation") // RemoteApiOptions.credentials is deprecated. + public void setUp() throws Exception { + installer = + new RemoteApiInstaller() { + @Override + AppEngineClient login(RemoteApiOptions options) { + return new AppEngineClient(options, ImmutableList.of(), getAppId()) { + @Override + public Response get(String path) throws IOException { + return null; + } + + @Override + public Response post(String path, String mimeType, byte[] body) throws IOException { + return null; + } + + @Override + public LegacyResponse legacyGet(String path) throws IOException { + return null; + } + + @Override + public LegacyResponse legacyPost(String path, String mimeType, byte[] body) + throws IOException { + return null; + } + }; + } + + @Override + RemoteApiDelegate createDelegate( + RemoteApiOptions options, + RemoteApiClient client, + ApiProxy.Delegate originalDelegate) { + rpc = new MockRpc(client); + return RemoteApiDelegate.newInstance(rpc, options, originalDelegate); + } + + @Override + ApiProxy.Environment createEnv(RemoteApiOptions options, RemoteApiClient client) { + return new ToolEnvironment(getAppId(), "someone@google.com"); + } + }; + + RemoteApiOptions ignoredOptions = + new RemoteApiOptions() + .server("ignored.example.com", 1234) + .credentials("someone@google.com", "ignored"); + installer.install(ignoredOptions); + datastore = DatastoreServiceFactory.getDatastoreService(); + } + + @After + public void tearDown() throws Exception { + installer.uninstall(); + } + + protected String getAppId() { + return "test~appId"; + } + + @Test + public void testFlushMemcache() throws Exception { + rpc.addResponse(MemcacheServicePb.MemcacheFlushResponse.getDefaultInstance()); + + MemcacheServiceFactory.getMemcacheService().clearAll(); + + rpc.verifyNextRpc("memcache", "FlushAll"); + rpc.verifyNoMoreRpc(); + } + + @Test + public void testJavaException() throws Exception { + ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); + ObjectOutput out = new ObjectOutputStream(byteStream); + out.writeObject(new RuntimeException("an exception")); + out.close(); + byte[] serializedException = byteStream.toByteArray(); + + RemoteApiPb.Response response = + RemoteApiPb.Response.newBuilder() + .setJavaException(ByteString.copyFrom(serializedException)) + .build(); + rpc.addResponse(response); + + RuntimeException e = assertThrows(RuntimeException.class, () -> datastore.get(KeyFactory.createKey("Something", 123))); + assertThat(e).hasMessageThat().isEqualTo("an exception"); + } + + @Test + public void testPythonException() throws Exception { + RemoteApiPb.Response response = + RemoteApiPb.Response.newBuilder() + .setException(ByteString.copyFromUtf8("pickle goes here")) + .build(); + rpc.addResponse(response); + + ApiProxy.ApiProxyException e = assertThrows(ApiProxy.ApiProxyException.class, () -> datastore.get(KeyFactory.createKey("Something", 123))); + assertThat(e).hasMessageThat().contains("response was a python exception:\n"); + assertThat(e).hasMessageThat().contains("pickle goes here"); + } + + @Test + public void testQueryReturningAllResultsInOneRpcCall() throws Exception { + replyToQuery(new Entity("Foo"), new Entity("Foo")); + + Query query = new Query("Foo"); + List result = datastore.prepare(query).asList(FetchOptions.Builder.withLimit(10)); + assertEquals(2, result.size()); + + verifyCalledRunQuery("Foo", null); + rpc.verifyNoMoreRpc(); + } + + /** + * Verifies that we can run a query that requires 3 RPC calls. Note that because queries run + * async, the RPC calls overlap with verification! + */ + @Test + public void testQueryReturningResultInThreeRpcCalls() throws Exception { + DatastoreV3Pb.CompiledCursor cursor1 = makeFakeCursor("cursor1"); + DatastoreV3Pb.CompiledCursor cursor2 = makeFakeCursor("cursor2"); + + Query query = new Query("Foo"); + + // Start RPC #1, which runs async. + + replyToQuery(cursor1, new Entity("Foo")); // needed by RPC #1 + Iterator result = datastore.prepare(query).asIterator(); + + // Wait for RPC #1. When it finishes, RPC #2 may start. + + replyToQuery(cursor2, new Entity("Foo"), new Entity("Foo")); // needed by RPC #2 + assertTrue(result.hasNext()); + verifyCalledRunQuery("Foo", null); // check RPC #1 + + // Skip one result from RPC #1 + + result.next(); + + // Wait for RPC #2, and RPC #3 might start. + + replyToQuery(new Entity("Foo")); // needed by RPC #3 + assertTrue(result.hasNext()); + verifyCalledRunQuery("Foo", cursor1); // check RPC #2 (Uses RunQuery, not Next.) + + // Skip two results from RPC #2 + + result.next(); + assertTrue(result.hasNext()); + result.next(); + + // Wait for RPC #3. No more RPC's should start. + + assertTrue(result.hasNext()); + verifyCalledRunQuery("Foo", cursor2); // check RPC #3 + rpc.verifyNoMoreRpc(); + + // Reading the rest of the query results shouldn't require any more RPCs. + + result.next(); + assertFalse(result.hasNext()); + + rpc.verifyNoMoreRpc(); + } + + @Test + public void testTransactionThatInsertsOneEntityWithNewId() throws Exception { + Entity entityToInsert = new Entity("Foo"); + + // Starting a transaction using the remote API doesn't require an RPC. + + Transaction tx = datastore.beginTransaction(); + rpc.verifyNoMoreRpc(); + + // In a transaction, put() calls don't happen immediately, but we need + // an RPC to allocate an id for the new Entity (if it doesn't have one already). + + Key newKey = KeyFactory.createKey("Foo", 123); + replyToGetIds(newKey); + datastore.put(entityToInsert); + rpc.verifyNextRpc("remote_datastore", "GetIDs"); + + assertEquals(newKey, entityToInsert.getKey()); + + // Committing the transaction performs the RPC. + + rpc.verifyNoMoreRpc(); + replyToRemoteTransaction(); + + tx.commit(); + + CommitChecker checker = verifyCommitRequest(); + checker.checkPreconditions(); // none + checker.checkPuts(entityToInsert); + checker.checkDeletes(); // none + } + + @Test + public void testTransactionThatUpdatesOneProperty() throws Exception { + + // The beginning of the transaction is handled locally, without doing any RPC call. + + Transaction tx = datastore.beginTransaction(); + rpc.verifyNoMoreRpc(); + + // When we get() the entity, the RPC is passed through. + + Entity entityReturnedByRpc = new Entity("Foo", "hello"); + replyToGetWithEntity(entityReturnedByRpc); + + Entity entity = datastore.get(KeyFactory.createKey("Foo", "hello")); + + rpc.verifyNextRpc("datastore_v3", "Get"); + assertEquals(entityReturnedByRpc, entity); + + entity.setProperty("bar", 123); + + // The put() call is remembered locally (the datastore won't be updated until commit). + + datastore.put(entity); + rpc.verifyNoMoreRpc(); + + // commit() uses a special RPC call to do the transaction all at once. + + replyToRemoteTransaction(); + + tx.commit(); + + CommitChecker checker = verifyCommitRequest(); + checker.checkPreconditions(entityReturnedByRpc); + checker.checkPuts(entity); + checker.checkDeletes(); // none + + rpc.verifyNoMoreRpc(); + } + + @Test + public void testTransactionThatDeletesAnEntity() throws Exception { + + Key keyToDelete = KeyFactory.createKey("Foo", "hello"); + + Transaction tx = datastore.beginTransaction(); + + // Do the delete. No RPC now, because deletes should be saved until the transaction commits. + + datastore.delete(keyToDelete); + rpc.verifyNoMoreRpc(); + + // Do the commit. + + replyToRemoteTransaction(); + + tx.commit(); + + CommitChecker checker = verifyCommitRequest(); + checker.checkPreconditions(); // none + checker.checkPuts(); // none + checker.checkDeletes(keyToDelete); + } + + @Test + public void testGetsAreCachedInTransaction() throws Exception { + + Key keyToGet = KeyFactory.createKey("Foo", "hello"); + Entity entityReturnedByRpc = new Entity("Foo", "hello"); + + Transaction tx = datastore.beginTransaction(); + + // First get does an rpc + + replyToGetWithEntity(entityReturnedByRpc); + + Entity firstEntity = datastore.get(keyToGet); + + rpc.verifyNextRpc("datastore_v3", "Get"); + assertEquals(entityReturnedByRpc, firstEntity); + + // Second get skips the rpc and returns the same entity + + Entity secondEntity = datastore.get(keyToGet); + rpc.verifyNoMoreRpc(); + + assertNotSame("entities shouldn't be the same", firstEntity, secondEntity); + assertEquals("entities should be equal", firstEntity, secondEntity); + + // The commit should contain one precondition + + replyToRemoteTransaction(); + + tx.commit(); + + CommitChecker checker = verifyCommitRequest(); + checker.checkPreconditions(entityReturnedByRpc); + checker.checkPuts(); // none + checker.checkDeletes(); // none + } + + @Test + public void testTransactionThatGetsNonexistentEntity() throws Exception { + + Key keyToGet = KeyFactory.createKey("Foo", "hello"); + + Transaction tx = datastore.beginTransaction(); + + // The first Get does an rpc and finds out there's no entity in the datastore. + + replyToGetWithMissing(keyToGet); + + EntityNotFoundException e1 = assertThrows(EntityNotFoundException.class, () -> datastore.get(keyToGet)); + assertThat(e1.getKey()).isEqualTo(keyToGet); + + rpc.verifyNextRpc("datastore_v3", "Get"); + + // The second Get skips the rpc and throws the same exception. + + EntityNotFoundException e2 = assertThrows(EntityNotFoundException.class, () -> datastore.get(keyToGet)); + assertThat(e2.getKey()).isEqualTo(keyToGet); + + rpc.verifyNoMoreRpc(); + + // The commit contains a precondition that the entity doesn't exist + + replyToRemoteTransaction(); + + tx.commit(); + + CommitChecker checker = verifyCommitRequest(); + checker.checkNonexistentEntityPreconditions(keyToGet); + checker.checkPuts(); // none + checker.checkDeletes(); // none + } + + @Test + public void testAncestorQueryInTransaction() throws Exception { + Key parentKey = KeyFactory.createKey("Parent", 123); + Key childKey = KeyFactory.createKey(parentKey, "Child", 456); + Entity childEntity = new Entity(childKey); + + Query query = new Query("Child"); + query.setAncestor(parentKey); + + Transaction tx = datastore.beginTransaction(); + PreparedQuery preparedQuery = datastore.prepare(tx, query); + + rpc.verifyNoMoreRpc(); + + Entity witness = replyToTxQuery(null, childEntity); + List result = preparedQuery.asList(FetchOptions.Builder.withLimit(10)); + + assertEquals(1, result.size()); + // The list pulls results lazily, so don't verify the rpc until after we've + // called size(). + verifyCalledTxQuery("Child", null); + assertEquals(childKey, result.get(0).getKey()); + + replyToRemoteTransaction(); + tx.commit(); + + CommitChecker checker = verifyCommitRequest(); + checker.checkPreconditions(witness); + checker.checkPuts(); // none + checker.checkDeletes(); // none + + rpc.verifyNoMoreRpc(); + } + + /** + * Verifies that we can run a query in a transaction that requires 2 RPC calls. Note that because + * queries run async, the RPC calls overlap with verification! + */ + @Test + public void testTwoPartAncestorQueryInTransaction() throws Exception { + DatastoreV3Pb.CompiledCursor cursor1 = makeFakeCursor("cursor1"); + + Key parentKey = KeyFactory.createKey("Parent", 123); + Key child1Key = KeyFactory.createKey(parentKey, "Child", 456); + Entity child1Entity = new Entity(child1Key); + Key child2Key = KeyFactory.createKey(parentKey, "Child", 457); + Entity child2Entity = new Entity(child2Key); + + Query query = new Query("Child"); + query.setAncestor(parentKey); + + Transaction tx = datastore.beginTransaction(); + PreparedQuery preparedQuery = datastore.prepare(tx, query); + + rpc.verifyNoMoreRpc(); + + // start RPC #1 (async) + + Entity witness1 = replyToTxQuery(cursor1, child1Entity); // needed by RPC #1 + Iterator it = preparedQuery.asIterator(); + + // wait for RPC #1 and start RPC #2 + + Entity witness2 = replyToTxQuery(null, child2Entity); // needed by RPC #2 + assertTrue(it.hasNext()); + verifyCalledTxQuery("Child", null); // check RPC #1 + assertEquals(child1Key, it.next().getKey()); + + // wait for RPC #2 + + boolean unused = it.hasNext(); + verifyCalledTxQuery("Child", cursor1); // check RPC #2 + rpc.verifyNoMoreRpc(); + + // check second result + + assertTrue(it.hasNext()); + assertEquals(child2Key, it.next().getKey()); + assertFalse(it.hasNext()); + + rpc.verifyNoMoreRpc(); + + // commit + + replyToRemoteTransaction(); + tx.commit(); + + CommitChecker checker = verifyCommitRequest(); + checker.checkPreconditions(witness1, witness2); + checker.checkPuts(); // none + checker.checkDeletes(); // none + } + + @Test + public void testRollbackTransaction() throws Exception { + Transaction tx = datastore.beginTransaction(); + rpc.verifyNoMoreRpc(); + + tx.rollback(); + rpc.verifyNoMoreRpc(); + } + + @Test + public void testNoRemoteDatastore() throws Exception { + String property = "com.google.appengine.devappserver2"; + String oldProperty = System.getProperty(property); + try { + System.setProperty(property, "true"); + doTestNoRemoteDatastore(); + } finally { + if (oldProperty == null) { + System.clearProperty(property); + } else { + System.setProperty(property, oldProperty); + } + } + } + + @SuppressWarnings( + "deprecation") // RemoteApiOptions.credentials is deprecated, OK to use in tests. + private void doTestNoRemoteDatastore() throws Exception { + RemoteApiOptions remoteApiOptions = + new RemoteApiOptions().credentials("someone@google.com", "password123"); + RemoteApiDelegate delegate = RemoteApiDelegate.newInstance(rpc, remoteApiOptions, null); + ApiProxy.Environment fakeEnvironment = new ToolEnvironment(getAppId(), "someone@google.com"); + byte[] fakeRequest = {1, 2, 3, 4}; + byte[] fakeResponse = {5, 6, 7, 8}; + rpc.addResponse(fakeResponse); + byte[] unused = + delegate.makeSyncCall( + fakeEnvironment, RemoteDatastore.DATASTORE_SERVICE, "Commit", fakeRequest); + rpc.verifyNextRpc(RemoteDatastore.DATASTORE_SERVICE, "Commit"); + rpc.verifyNoMoreRpc(); + } + + // === end of tests === + + @SuppressWarnings("deprecation") // CompiledCursor.Position.start_key is deprecated but we need to + // set it here for testing. + private static DatastoreV3Pb.CompiledCursor makeFakeCursor(String name) { + DatastoreV3Pb.CompiledCursor.Builder result = DatastoreV3Pb.CompiledCursor.newBuilder(); + // DatastoreV3Pb (proto2api) uses String for string fields. + result.getPositionBuilder().setStartKey(ByteString.copyFromUtf8(name)); + return result.build(); + } + + private void replyToQuery(Entity... entities) { + replyToQuery(null, entities); + } + + private void replyToQuery(DatastoreV3Pb.CompiledCursor cursor, Entity... entities) { + DatastoreV3Pb.QueryResult.Builder resultPb = DatastoreV3Pb.QueryResult.newBuilder(); + for (Entity entity : entities) { + resultPb.addResult(entityToProto2(entity)); + } + if (cursor != null) { + resultPb.setMoreResults(true); + resultPb.setCompiledCursor(cursor); + } else { + resultPb.setMoreResults(false); + } + rpc.addResponse(resultPb.build()); + } + + private Entity replyToTxQuery(DatastoreV3Pb.CompiledCursor cursor, Entity... entities) { + RemoteApiPb.TransactionQueryResult.Builder resultPb = + RemoteApiPb.TransactionQueryResult.newBuilder(); + DatastoreV3Pb.QueryResult.Builder queryResultPb = resultPb.getResultBuilder(); + for (Entity entity : entities) { + queryResultPb.addResult(entityToProto2(entity)); + } + if (cursor != null) { + queryResultPb.setMoreResults(true); + queryResultPb.setCompiledCursor(cursor); + } else { + queryResultPb.setMoreResults(false); + } + // Make a somewhat arbitrary witness. + Entity witness = new Entity("__entity_group__", "foo", entities[0].getKey()); + resultPb + .setEntityGroupKey(entityToProto2(witness).getKey()) + .setEntityGroup(entityToProto2(witness)); + rpc.addResponse(resultPb.build()); + return witness; + } + + private void replyToGetWithMissing(Key missingKey) { + DatastoreV3Pb.GetResponse.Builder response = DatastoreV3Pb.GetResponse.newBuilder(); + response.addEntityBuilder().setKey(keyToRefProto2(missingKey)); + rpc.addResponse(response.build()); + } + + private void replyToGetWithEntity(Entity entity) { + DatastoreV3Pb.GetResponse.Builder response = DatastoreV3Pb.GetResponse.newBuilder(); + response.addEntityBuilder().setEntity(entityToProto2(entity)); + rpc.addResponse(response.build()); + } + + private void replyToGetIds(Key... updatedKeys) { + DatastoreV3Pb.PutResponse.Builder response = DatastoreV3Pb.PutResponse.newBuilder(); + for (Key key : updatedKeys) { + response.addKey(keyToRefProto2(key)); + } + rpc.addResponse(response.build()); + } + + private void replyToRemoteTransaction() { + DatastoreV3Pb.PutResponse response = DatastoreV3Pb.PutResponse.getDefaultInstance(); + rpc.addResponse(response.toByteArray()); + } + + @CanIgnoreReturnValue + private DatastoreV3Pb.Query verifyCalledRunQuery( + String expectedKind, DatastoreV3Pb.CompiledCursor expectedCursor) { + RemoteApiPb.Request wrappedRequest = rpc.verifyNextRpc("datastore_v3", "RunQuery"); + DatastoreV3Pb.Query query = verifyQuery(wrappedRequest, expectedKind, expectedCursor); + // not in a transaction + assertFalse("query shouldn't be in a transaction", query.hasTransaction()); + + return query; + } + + @CanIgnoreReturnValue + private DatastoreV3Pb.Query verifyCalledTxQuery( + String expectedKind, DatastoreV3Pb.CompiledCursor expectedCursor) { + RemoteApiPb.Request wrappedRequest = rpc.verifyNextRpc("remote_datastore", "TransactionQuery"); + DatastoreV3Pb.Query query = verifyQuery(wrappedRequest, expectedKind, expectedCursor); + + return query; + } + + @SuppressWarnings("deprecation") // Testing deprecated Query.compile field. + private static DatastoreV3Pb.Query verifyQuery( + RemoteApiPb.Request wrappedRequest, + String expectedKind, + DatastoreV3Pb.CompiledCursor expectedCursor) { + DatastoreV3Pb.Query query; + try { + query = + DatastoreV3Pb.Query.parseFrom( + wrappedRequest.getRequest(), ExtensionRegistry.getEmptyRegistry()); + } catch (InvalidProtocolBufferException e) { + throw new RuntimeException(e); + } + + assertEquals("query on unexpected kind", expectedKind, query.getKind()); + assertWithMessage("Compile flag not set for query").that(query.getCompile()).isTrue(); + assertEquals("query with unexpected offset", 0, query.getOffset()); + if (expectedCursor != null) { + assertEquals("cursor doesn't match", expectedCursor, query.getCompiledCursor()); + } else { + assertFalse(query.hasCompiledCursor()); + } + + return query; + } + + private CommitChecker verifyCommitRequest() { + RemoteApiPb.Request wrappedRequest = rpc.verifyNextRpc("remote_datastore", "Transaction"); + return new CommitChecker(wrappedRequest); + } + + private static OnestoreEntity.EntityProto entityToProto2(Entity entity) { + return EntityTranslator.convertToPb(entity); + } + + private static OnestoreEntity.Reference keyToRefProto2(Key key) { + try { + String encoded = KeyFactory.keyToString(key); + byte[] bytes = base64Url().decode(encoded); + return OnestoreEntity.Reference.parseFrom(bytes, ExtensionRegistry.getEmptyRegistry()); + } catch (Exception e) { + throw new RuntimeException("can't convert key to reference", e); + } + } + + /** + * Accepts RPC calls and returns the appropriate fake value. Handles async calls by waiting before + * verifying. + */ + static class MockRpc extends RemoteRpc { + + Queue receivedRequests; + Queue responsesToSend; + + MockRpc(RemoteApiClient client) { + super(client); + receivedRequests = new LinkedBlockingQueue(); + responsesToSend = new LinkedBlockingQueue(); + } + + void addResponse(RemoteApiPb.Response response) { + responsesToSend.add(response); + } + + void addResponse(byte[] bytes) { + RemoteApiPb.Response response = + RemoteApiPb.Response.newBuilder().setResponse(ByteString.copyFrom(bytes)).build(); + + addResponse(response); + } + + void addResponse(Message result) { + addResponse(result.toByteArray()); + } + + @Override + RemoteApiPb.Response callImpl(RemoteApiPb.Request requestProto) { + receivedRequests.add(requestProto); + return responsesToSend.remove(); + } + + /** + * Checks that at least one RPC happened and returns its value. In the case of async RPC, you + * must arrange to wait until the async call finishes before calling this method (typically by + * getting the value from the Future). + */ + RemoteApiPb.Request verifyNextRpc() { + return receivedRequests.remove(); + } + + @CanIgnoreReturnValue + RemoteApiPb.Request verifyNextRpc(String expectedService, String expectedMethod) { + RemoteApiPb.Request request = verifyNextRpc(); + assertWithMessage("unexpected method call") + .that(request.getServiceName() + "." + request.getMethod()) + .isEqualTo(expectedService + "." + expectedMethod); + return request; + } + + /** Verify that no RPC happened. (Waits a bit to make sure an async RPC isn't in progress.) */ + void verifyNoMoreRpc() throws InterruptedException { + // need to wait in the case of async calls + Thread.sleep(1000); + assertThat(receivedRequests).isEmpty(); + assertThat(responsesToSend).isEmpty(); + } + } + + /** Verifies the fields in a request to commit a transaction using the remote API. */ + static class CommitChecker { + private final RemoteApiPb.TransactionRequest actualRequest; + + CommitChecker(RemoteApiPb.Request wrappedRequest) { + try { + actualRequest = + RemoteApiPb.TransactionRequest.parseFrom( + wrappedRequest.getRequest(), ExtensionRegistry.getEmptyRegistry()); + } catch (InvalidProtocolBufferException e) { + throw new RuntimeException(e); + } + } + + void checkPreconditions(Entity... expectedEntities) { + + List> expected = Lists.newArrayList(); + for (Entity entity : expectedEntities) { + ByteString key = entityToProto2(entity).getKey().toByteString(); + ByteString hash = getSha1Hash(entity); + expected.add(new SimpleImmutableEntry<>(key, hash)); + } + + checkPreconditions(expected); + } + + private void checkPreconditions(List> expected) { + List> actual = Lists.newArrayList(); + for (RemoteApiPb.TransactionRequest.Precondition precondition : + actualRequest.getPreconditionList()) { + ByteString key = precondition.getKey().toByteString(); + ByteString hash = precondition.hasHash() ? precondition.getHash() : null; + actual.add(new SimpleImmutableEntry<>(key, hash)); + } + + assertWithMessage("commit call doesn't have the right preconditions") + .that(actual) + .containsExactlyElementsIn(expected); + } + + void checkNonexistentEntityPreconditions(Key... expectedKeys) { + List> expected = Lists.newArrayList(); + for (Key key : expectedKeys) { + ByteString keyBytes = keyToRefProto2(key).toByteString(); + expected.add(new SimpleImmutableEntry<>(keyBytes, (ByteString) null)); + } + checkPreconditions(expected); + } + + void checkPuts(Entity... expectedPuts) { + assertWithMessage("commit call doesn't put() the right entities") + .that(createEntities(actualRequest.getPuts().getEntityList())) + .containsExactlyElementsIn(expectedPuts); + } + + void checkDeletes(Key... expected) { + assertWithMessage("commit call doesn't delete the right entities") + .that(createKeys(actualRequest.getDeletes().getKeyList())) + .containsExactlyElementsIn(expected); + } + + private static ByteString getSha1Hash(Entity entity) { + byte[] bytes = entityToProto2(entity).toByteArray(); + return ByteString.copyFrom(Hashing.sha1().hashBytes(bytes, 0, bytes.length).asBytes()); + } + + private static List createEntities(Iterable entities) { + List result = Lists.newArrayList(); + for (OnestoreEntity.EntityProto entityPb : entities) { + result.add(EntityTranslator.createFromPb(entityPb)); + } + return result; + } + + private static List createKeys(Iterable keys) { + List result = Lists.newArrayList(); + for (OnestoreEntity.Reference keyProto : keys) { + result.add(createKey(keyProto)); + } + return result; + } + + private static Key createKey(OnestoreEntity.Reference keyPb) { + // KeyTranslator isn't public but we can call it indirectly through KeyFactory + return KeyFactory.stringToKey(base64Url().omitPadding().encode(keyPb.toByteArray())); + } + } +} diff --git a/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/RemoteApiInstallerAllThreadsTest.java b/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/RemoteApiInstallerAllThreadsTest.java new file mode 100644 index 000000000..481027c86 --- /dev/null +++ b/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/RemoteApiInstallerAllThreadsTest.java @@ -0,0 +1,102 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.tools.remoteapi; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; + +import com.google.apphosting.api.ApiProxy; +import java.io.IOException; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import org.apache.http.cookie.Cookie; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Test for {@link RemoteApiInstaller#installOnAllThreads(RemoteApiOptions)}. + * This method leaves behind static state in {@link ApiProxy} that can't be + * changed, so it's run as a separate test. + */ +@RunWith(JUnit4.class) +public class RemoteApiInstallerAllThreadsTest { + + @Test + public void testInstallOnAllThreads() throws Exception { + // Confirm that we're starting with a clean state. + assertNull(ApiProxy.getCurrentEnvironment()); + assertNull(ApiProxy.getDelegate()); + + final RemoteApiInstaller installer = newInstaller("appId"); + installer.installOnAllThreads(createDummyOptions()); + + // Inspect the ApiProxy from this thread. + assertNotNull(ApiProxy.getDelegate()); + assertNotNull(ApiProxy.getCurrentEnvironment()); + + ExecutorService executor = Executors.newSingleThreadExecutor(); + + // Inspect the ApiProxy from another thread. + executor.submit(() -> { + assertNotNull(ApiProxy.getDelegate()); + assertNotNull(ApiProxy.getCurrentEnvironment()); + }).get(); + + // Can't re-install on all threads. + assertThrows(IllegalStateException.class, () -> installer.installOnAllThreads(createDummyOptions())); + + // Can't re-install on all threads from a separate thread. + executor.submit(() -> { + assertThrows(IllegalStateException.class, () -> installer.installOnAllThreads(createDummyOptions())); + }).get(); + + // Can't install for a single thread. + assertThrows(IllegalStateException.class, () -> installer.install(createDummyOptions())); + + // Can't install for a single separate thread. + executor.submit(() -> { + assertThrows(IllegalStateException.class, () -> installer.install(createDummyOptions())); + }).get(); + + // Can't uninstall. + assertThrows(IllegalArgumentException.class, () -> installer.uninstall()); + + // Can't uninstall from a separate thread. + executor.submit(() -> { + assertThrows(IllegalArgumentException.class, () -> installer.uninstall()); + }).get(); + } + + private static RemoteApiOptions createDummyOptions() { + return new RemoteApiOptions() + .server("localhost", 8080) + .credentials("this", "that"); + } + + private static RemoteApiInstaller newInstaller(final String remoteAppId) { + return new RemoteApiInstaller() { + @Override + String getAppIdFromServer(List authCookies, RemoteApiOptions options) + throws IOException { + return remoteAppId; + } + }; + } +} diff --git a/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/RemoteApiInstallerTest.java b/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/RemoteApiInstallerTest.java new file mode 100644 index 000000000..12402673e --- /dev/null +++ b/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/RemoteApiInstallerTest.java @@ -0,0 +1,325 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.tools.remoteapi; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import com.google.appengine.tools.remoteapi.testing.StubCredential; +import com.google.apphosting.api.ApiProxy; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import org.apache.http.cookie.Cookie; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Verifies some of the methods in {@link RemoteApiInstaller}. + * + */ +@RunWith(JUnit4.class) +public class RemoteApiInstallerTest { + + @Before + public void setUp() throws Exception { + // Start with no environment (simulating a standalone setup). Individual tests may change this. + ApiProxy.setEnvironmentForCurrentThread(null); + } + + /** + * Verifies that we can parse the YAML map returned by the RemoteApiServlet. + * (This map contains the app id.) + */ + @Test + public void testParseYamlMap() throws Exception { + Map output = RemoteApiInstaller.parseYamlMap( + " {rtok: null, app_id: my-app-id}\n\n"); + + assertEquals("null", output.get("rtok")); + assertEquals("my-app-id", output.get("app_id")); + assertThat(output).hasSize(2); + + // now try an HR app + output = RemoteApiInstaller.parseYamlMap( + " {rtok: null, app_id: s~my-app-id}\n\n"); + + assertEquals("null", output.get("rtok")); + assertEquals("s~my-app-id", output.get("app_id")); + assertThat(output).hasSize(2); + + // the fields may be quoted using double strings. + output = RemoteApiInstaller.parseYamlMap( + "{app_id: \"s~go-tee\", rtok: \"0\"}"); + + assertEquals("s~go-tee", output.get("app_id")); + assertEquals("0", output.get("rtok")); + assertThat(output).hasSize(2); + + // mismatched quotes should fail to parse. + output = RemoteApiInstaller.parseYamlMap( + "{app_id: 's~go-tee\", rtok: \"0}"); + + assertThat(output).isEmpty(); + + output = RemoteApiInstaller.parseYamlMap( + "{app_id: s~go-tee', rtok: \"0'}"); + + assertThat(output).isEmpty(); + } + + /** + * Verifies that we can parse the YAML map returned by the RemoteApiServlet + * for an app running on internal infrastructure. + * (This map contains the app id.) + */ + @Test + public void testParseYamlMapInternal() throws Exception { + Map output = RemoteApiInstaller.parseYamlMap( + " {rtok: 'null', app_id: 'google.com:my-app-id'}\n\n"); + + assertEquals("null", output.get("rtok")); + assertEquals("google.com:my-app-id", output.get("app_id")); + assertThat(output).hasSize(2); + + // now try an HR app + output = RemoteApiInstaller.parseYamlMap( + " {rtok: 'null', app_id: 's~google.com:my-app-id'}\n\n"); + + assertEquals("null", output.get("rtok")); + assertEquals("s~google.com:my-app-id", output.get("app_id")); + assertThat(output).hasSize(2); + } + + @Test + public void testParseSerializedCredentials() throws Exception { + String credentials = + """ + + # example credential file + + host=somehost.prom.corp.google.com + email=foo@google.com + + cookie=SACSID=ASDFASDF + cookie=SOMETHING=else + """; + + List cookies = RemoteApiInstaller.parseSerializedCredentials("foo@google.com", + "somehost.prom.corp.google.com", credentials); + + Cookie cookie = cookies.get(0); + assertEquals("SACSID", cookie.getName()); + assertEquals("ASDFASDF", cookie.getValue()); + + cookie = cookies.get(1); + assertEquals("SOMETHING", cookie.getName()); + assertEquals("else", cookie.getValue()); + + assertEquals(2, cookies.size()); + } + + @Test + @SuppressWarnings("rawtypes") // ApiProxy.getDelegate() returns a raw type. + public void testInstallationStandalone() throws Exception { + assertNull(ApiProxy.getCurrentEnvironment()); + + // On Install, it should set a ToolsEnvironment on the Thread. + RemoteApiInstaller installer = install("yar"); + assertTrue(ToolEnvironment.class.equals(ApiProxy.getCurrentEnvironment().getClass())); + assertThat(ApiProxy.getDelegate()).isInstanceOf(ThreadLocalDelegate.class); + + // Removes the Environment on uninstall. + installer.uninstall(); + assertNull(ApiProxy.getCurrentEnvironment()); + assertThat(ApiProxy.getDelegate()).isInstanceOf(ThreadLocalDelegate.class); + ThreadLocalDelegate tld = (ThreadLocalDelegate) ApiProxy.getDelegate(); + + installer = install("yar"); + assertSame(tld, ApiProxy.getDelegate()); + installer.uninstall(); + assertSame(tld, ApiProxy.getDelegate()); + } + + @Test + public void testInstallationHosted() throws Exception { + // This is kind of cheating. The RemoteApiInstaller checks for the presence of an Environment + // to determine if it's running standalone or hosted. We'll trick it by putting a + // ToolsEnvironment in there. + ApiProxy.setEnvironmentForCurrentThread(new ToolEnvironment("unused-appid", "unused-email")); + + RemoteApiInstaller installer = install("yar"); + + // Should have left the existing Environment in place, but added an attribute to override the + // app ids in Datastore keys. + assertTrue(ToolEnvironment.class.equals(ApiProxy.getCurrentEnvironment().getClass())); + assertEquals( + "yar", + ApiProxy.getCurrentEnvironment().getAttributes().get( + RemoteApiInstaller.DATASTORE_APP_ID_OVERRIDE_KEY)); + + // The app id override is removed after uninstalling. + installer.uninstall(); + assertThat(ApiProxy.getCurrentEnvironment().getAttributes()) + .doesNotContainKey(RemoteApiInstaller.DATASTORE_APP_ID_OVERRIDE_KEY); + } + + @Test + public void testInstallationHostedWithExistingAppIdOverride() throws Exception { + // This is kind of cheating. The RemoteApiInstaller checks for the presence of an Environment + // to determine if it's running standalone or hosted. We'll trick it by putting a + // ToolsEnvironment in there. + ApiProxy.setEnvironmentForCurrentThread(new ToolEnvironment("unused-appid", "unused-email")); + + // Simulate there already being an override. This is not something we really expect, but we'll + // make sure it works. + ApiProxy.getCurrentEnvironment().getAttributes().put( + RemoteApiInstaller.DATASTORE_APP_ID_OVERRIDE_KEY, "somePreexistingOverride"); + + RemoteApiInstaller installer = install("yar"); + + // Should have left the existing Environment in place, but changed the app id override. + assertTrue(ToolEnvironment.class.equals(ApiProxy.getCurrentEnvironment().getClass())); + assertEquals( + "yar", + ApiProxy.getCurrentEnvironment().getAttributes().get( + RemoteApiInstaller.DATASTORE_APP_ID_OVERRIDE_KEY)); + + // The app id override is removed after uninstalling and should restore the old override. + installer.uninstall(); + assertEquals( + "somePreexistingOverride", + ApiProxy.getCurrentEnvironment().getAttributes().get( + RemoteApiInstaller.DATASTORE_APP_ID_OVERRIDE_KEY)); + } + + @Test + @SuppressWarnings("rawtypes") + public void testInstallationOnDifferentThreads() throws Exception { + // Install in the test thread. + RemoteApiInstaller installer1 = install("yar"); + final ApiProxy.Delegate tld = ApiProxy.getDelegate(); + assertThat(tld).isInstanceOf(ThreadLocalDelegate.class); + + ExecutorService svc = Executors.newSingleThreadExecutor(); + + // Install in an alternate thread. + Callable callable = () -> { + RemoteApiInstaller installer = install("yar"); + assertSame(tld, ApiProxy.getDelegate()); + assertNotNull(((ThreadLocalDelegate) ApiProxy.getDelegate()).getDelegateForThread()); + return installer; + }; + final RemoteApiInstaller installer2 = svc.submit(callable).get(); + + // Uninstall in the test thread. + installer1.uninstall(); + assertSame(tld, ApiProxy.getDelegate()); + assertNull(((ThreadLocalDelegate) ApiProxy.getDelegate()).getDelegateForThread()); + + // Uninstall in the alternate thread. + @SuppressWarnings("unused") // go/futurereturn-lsc + Future possiblyIgnoredError = + svc.submit( + () -> { + assertThat(((ThreadLocalDelegate) ApiProxy.getDelegate()).getDelegateForThread()) + .isNotNull(); + installer2.uninstall(); + assertThat(((ThreadLocalDelegate) ApiProxy.getDelegate()).getDelegateForThread()) + .isNull(); + return null; + }); + assertSame(tld, ApiProxy.getDelegate()); + assertNull(((ThreadLocalDelegate) ApiProxy.getDelegate()).getDelegateForThread()); + } + + @Test + public void testValidateOptionsNullHostname() { + RemoteApiOptions options = new RemoteApiOptions() + .server(null, 8080) + .credentials("email", "password"); + assertThrows(IllegalArgumentException.class, () -> new RemoteApiInstaller().validateOptions(options)); + } + + @Test + public void testValidateOptionsNoCredentials() { + RemoteApiOptions options = new RemoteApiOptions() + .server("hostname", 8080); + assertThrows(IllegalArgumentException.class, () -> new RemoteApiInstaller().validateOptions(options)); + } + + @Test + public void testValidateOptionsPasswordCredentials() { + RemoteApiOptions options = new RemoteApiOptions() + .server("hostname", 8080) + .credentials("email", "password"); + new RemoteApiInstaller().validateOptions(options); + } + + @Test + public void testValidateOptionsDevAppServerCredentials() { + RemoteApiOptions options = new RemoteApiOptions() + .server("hostname", 8080) + .useDevelopmentServerCredential(); + new RemoteApiInstaller().validateOptions(options); + } + + @Test + public void testValidateOptionsOAuthCredentials() { + RemoteApiOptions options = new RemoteApiOptions() + .server("hostname", 8080) + .oauthCredential(new StubCredential()); + new RemoteApiInstaller().validateOptions(options); + } + + private static RemoteApiOptions createDummyOptions() { + return new RemoteApiOptions() + .server("localhost", 8080) + .credentials("this", "that"); + } + + private static RemoteApiInstaller newInstaller(final String remoteAppId) { + return new RemoteApiInstaller() { + @Override + String getAppIdFromServer(List authCookies, RemoteApiOptions options) + throws IOException { + return remoteAppId; + } + }; + } + + RemoteApiInstaller install(final String remoteAppId) { + RemoteApiInstaller installer = newInstaller(remoteAppId); + try { + installer.install(createDummyOptions()); + } catch (IOException e) { + throw new RuntimeException(e); + } + return installer; + } +} diff --git a/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/RemoteApiOptionsTest.java b/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/RemoteApiOptionsTest.java new file mode 100644 index 000000000..1abf9230c --- /dev/null +++ b/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/RemoteApiOptionsTest.java @@ -0,0 +1,213 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.tools.remoteapi; + +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotSame; + +import com.google.api.client.testing.http.MockHttpTransport; +import com.google.appengine.tools.remoteapi.testing.StubCredential; +import com.google.apphosting.api.ApiProxy; +import com.google.apphosting.api.ApiProxy.Environment; +import com.google.common.collect.ImmutableList; +import java.io.File; +import java.io.InputStream; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.net.URL; +import java.nio.file.Files; +import java.util.Map; +import java.util.Objects; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Tests for {@link RemoteApiOptions}. In particular this ensures that if we add a new option we + * remember to update the copy method. + * + */ +@RunWith(JUnit4.class) +public class RemoteApiOptionsTest { + + private static class StubEnvironment implements Environment { + @Override + public boolean isLoggedIn() { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isAdmin() { + throw new UnsupportedOperationException(); + } + + @Override + public String getVersionId() { + throw new UnsupportedOperationException(); + } + + @Deprecated + @Override + public String getRequestNamespace() { + throw new UnsupportedOperationException(); + } + + @Override + public long getRemainingMillis() { + throw new UnsupportedOperationException(); + } + + @Override + public String getModuleId() { + throw new UnsupportedOperationException(); + } + + @Override + public String getEmail() { + throw new UnsupportedOperationException(); + } + + @Override + public String getAuthDomain() { + throw new UnsupportedOperationException(); + } + + @Override + public Map getAttributes() { + throw new UnsupportedOperationException(); + } + + @Override + public String getAppId() { + throw new UnsupportedOperationException(); + } + } + + /** + * Return a list of RemoteApiOptions objects such that for any given property, at least one of + * the objects has a non-default value for that property. + * This will need to be updated every time a new property is added to the class. + */ + private static ImmutableList nonDefaultOptions() { + RemoteApiOptions optionsWithPassword = new RemoteApiOptions() + .credentials("foo@bar.com", "mysecurepassword") + .datastoreQueryFetchSize(17) + .maxConcurrentRequests(23) + .maxHttpResponseSize(1729) + .remoteApiPath("/path/to/remote/api") + .server("somehostname", 8080); + RemoteApiOptions optionsWithCredentials = optionsWithPassword.copy() + .reuseCredentials("foo@bar.com", "myserializedcredentials"); + RemoteApiOptions optionsWithOAuthCredentials = optionsWithPassword.copy() + .oauthCredential(new StubCredential()) + .httpTransport(new MockHttpTransport()); + return ImmutableList.of(optionsWithPassword, optionsWithCredentials, + optionsWithOAuthCredentials); + } + + /** + * Test that {@link #nonDefaultOptions()} does return a non-default value somewhere for every + * property. The validity of the tests for {@link RemoteApiOptions#copy} depends on this. + */ + @Test + public void testNonDefaultOptions() throws Exception { + RemoteApiOptions defaultOptions = new RemoteApiOptions(); + for (Field field : RemoteApiOptions.class.getDeclaredFields()) { + if (!Modifier.isStatic(field.getModifiers())) { + boolean allSame = true; + for (RemoteApiOptions nonDefaultOptions : nonDefaultOptions()) { + allSame &= sameValueOfFieldIn(field, defaultOptions, nonDefaultOptions); + } + assertFalse(field.getName(), allSame); + } + } + } + + @Test + public void testDefaultCopy() { + RemoteApiOptions defaultOptions = new RemoteApiOptions(); + assertOptionsEqual(defaultOptions, defaultOptions.copy()); + } + + @Test + public void testCopyCopiesEverything() throws Exception { + for (RemoteApiOptions nonDefaultOptions : nonDefaultOptions()) { + RemoteApiOptions copy = nonDefaultOptions.copy(); + assertOptionsEqual(nonDefaultOptions, copy); + } + } + + @Test + public void testOAuthCredentialsSupportedOnAppEngineClient() throws Exception { + ApiProxy.setEnvironmentForCurrentThread(new StubEnvironment()); + URL url = + this.getClass() + .getClassLoader() + .getResource("com/google/appengine/tools/remoteapi/testdata/test.pkcs12"); + if (url == null) { + url = + this.getClass() + .getClassLoader() + .getResource( + "src/test/resources/com/google/appengine/tools/remoteapi/testdata/test.pkcs12"); + } + if (url == null) { + url = + this.getClass() + .getClassLoader() + .getResource( + "third_party/java_src/appengine_standard/remoteapi/src/test/resources/com/google/appengine/tools/remoteapi/testdata/test.pkcs12"); + } + File tempFile = File.createTempFile("test", ".pkcs12"); + tempFile.deleteOnExit(); + try (InputStream in = url.openStream()) { + Files.copy(in, tempFile.toPath(), REPLACE_EXISTING); + } + RemoteApiOptions options = new RemoteApiOptions(); + options.useServiceAccountCredential("foo@example.com", tempFile.getAbsolutePath()); + } + + private static void assertOptionsEqual(RemoteApiOptions x, RemoteApiOptions y) { + assertNotSame(x, y); + assertEquals(null, differingOption(x, y)); + // JUnit doesn't show the non-null value if assertNull fails. Grrr. + } + + /** + * Return the first option that differs between the two objects, or null if they have all the + * same options. + */ + private static String differingOption(RemoteApiOptions x, RemoteApiOptions y) { + for (Field field : RemoteApiOptions.class.getDeclaredFields()) { + if (!sameValueOfFieldIn(field, x, y)) { + return field.getName(); + } + } + return null; + } + + private static boolean sameValueOfFieldIn(Field field, RemoteApiOptions x, RemoteApiOptions y) { + try { + field.setAccessible(true); + return Objects.deepEquals(field.get(x), field.get(y)); + } catch (IllegalAccessException e) { + throw new AssertionError(e); + } + } +} diff --git a/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/RemoteDatastoreTest.java b/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/RemoteDatastoreTest.java new file mode 100644 index 000000000..820b43498 --- /dev/null +++ b/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/RemoteDatastoreTest.java @@ -0,0 +1,454 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.tools.remoteapi; + +import static com.google.common.truth.Truth.assertWithMessage; +import static com.google.common.truth.extensions.proto.ProtoTruth.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.when; + +import com.google.apphosting.datastore_bytes.proto2api.DatastoreV3Pb; +import com.google.common.collect.Lists; +import com.google.protobuf.ByteString; +import com.google.protobuf.ExtensionRegistry; +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.storage.onestore.v3_bytes.proto2api.OnestoreEntity; +import com.google.storage.onestore.v3_bytes.proto2api.OnestoreEntity.EntityProto; +import com.google.storage.onestore.v3_bytes.proto2api.OnestoreEntity.Reference; +import java.util.List; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +/** + * Test for {@link RemoteDatastore} + * + * @author maxr@google.com (Max Ross) + */ +@RunWith(JUnit4.class) +public class RemoteDatastoreTest { + + private static final String CLIENT_APP_ID = "client"; + private static final String TARGET_APP_ID = "target"; + + @Rule public final MockitoRule mocks = MockitoJUnit.rule(); + @Mock private RemoteRpc mockRemoteRpc; + + private RemoteDatastore remoteDatastore; + + private static final RewrittenReferenceHolder REFERENCE_HOLDER_1 = + new RewrittenReferenceHolder(1); + private static final RewrittenReferenceHolder REFERENCE_HOLDER_2 = + new RewrittenReferenceHolder(2); + private static final RewrittenReferenceHolder REFERENCE_HOLDER_3 = + new RewrittenReferenceHolder(3); + private static final RewrittenReferenceHolder REFERENCE_HOLDER_4 = + new RewrittenReferenceHolder(4); + private static final RewrittenReferenceHolder REFERENCE_HOLDER_5 = + new RewrittenReferenceHolder(5); + private static final RewrittenReferenceHolder REFERENCE_HOLDER_6 = + new RewrittenReferenceHolder(6); + + /** Wrapper around both formats of Reference and Entity. */ + private static class RewrittenReferenceHolder { + private final Reference clientReference; + private final Reference targetReference; + private final EntityProto clientEntity; + private final EntityProto targetEntity; + + private RewrittenReferenceHolder(int index) { + this.clientEntity = createEntityProto(CLIENT_APP_ID, index); + this.targetEntity = createEntityProto(TARGET_APP_ID, index); + this.clientReference = clientEntity.getKey(); + this.targetReference = targetEntity.getKey(); + } + } + + @Before + public void setUpRemoteDatastore() { + remoteDatastore = new RemoteDatastore(TARGET_APP_ID, mockRemoteRpc, new RemoteApiOptions()); + } + + @Test + public void testGetRewritesReferences() throws InvalidProtocolBufferException { + DatastoreV3Pb.GetRequest.Builder getRequest = DatastoreV3Pb.GetRequest.newBuilder(); + getRequest.addKey(REFERENCE_HOLDER_1.clientReference); + getRequest.addKey(REFERENCE_HOLDER_2.clientReference); + getRequest.addKey(REFERENCE_HOLDER_3.clientReference); + getRequest.addKey(REFERENCE_HOLDER_4.targetReference); // This key already has the target app. + getRequest.addKey(REFERENCE_HOLDER_5.clientReference); + + DatastoreV3Pb.GetRequest rewrittenGetRequest = + DatastoreV3Pb.GetRequest.newBuilder() + .addKey(REFERENCE_HOLDER_1.targetReference) + .addKey(REFERENCE_HOLDER_2.targetReference) + .addKey(REFERENCE_HOLDER_3.targetReference) + .addKey(REFERENCE_HOLDER_4.targetReference) + .addKey(REFERENCE_HOLDER_5.targetReference) + .build(); + + // A mix of Entities, Missing, and Deferred. + DatastoreV3Pb.GetResponse.Builder remoteRpcResponse = DatastoreV3Pb.GetResponse.newBuilder(); + remoteRpcResponse.addEntityBuilder().setKey(REFERENCE_HOLDER_1.targetReference); + remoteRpcResponse.addEntityBuilder().setEntity(REFERENCE_HOLDER_2.targetEntity); + remoteRpcResponse.addEntityBuilder().setEntity(REFERENCE_HOLDER_3.targetEntity); + remoteRpcResponse.addDeferred(REFERENCE_HOLDER_4.targetReference); + remoteRpcResponse.addEntityBuilder().setKey(REFERENCE_HOLDER_5.targetReference); + + expectRemoteRpcGet(rewrittenGetRequest, remoteRpcResponse.build()); + + // We should get a serialized version of the remoteRpcResponse. + DatastoreV3Pb.GetResponse actualResp = invokeGet(getRequest.build()); + assertThat(actualResp).isEqualTo(remoteRpcResponse.build()); + } + + @Test + public void testGetAndPutInTransaction() throws InvalidProtocolBufferException { + // Begin a transaction. + DatastoreV3Pb.Transaction transaction = beginTransaction(); + + // Issue a Get request. + DatastoreV3Pb.GetRequest getRequest1 = + DatastoreV3Pb.GetRequest.newBuilder() + .setTransaction(transaction) + .addKey(REFERENCE_HOLDER_1.clientReference) + .addKey(REFERENCE_HOLDER_2.clientReference) + .build(); + + DatastoreV3Pb.GetRequest rewrittenGetRequest1 = + DatastoreV3Pb.GetRequest.newBuilder() + .addKey(REFERENCE_HOLDER_1.targetReference) + .addKey(REFERENCE_HOLDER_2.targetReference) + .build(); + + // One found, one missing + DatastoreV3Pb.GetResponse.Builder remoteRpcGetResponse1 = + DatastoreV3Pb.GetResponse.newBuilder(); + remoteRpcGetResponse1.addEntityBuilder().setKey(REFERENCE_HOLDER_1.targetReference); + remoteRpcGetResponse1.addEntityBuilder().setEntity(REFERENCE_HOLDER_2.targetEntity); + + expectRemoteRpcGet(rewrittenGetRequest1, remoteRpcGetResponse1.build()); + + // We should get a serialized version of the remoteRpcResponse. + DatastoreV3Pb.GetResponse actualResp1 = invokeGet(getRequest1); + assertThat(actualResp1) + .ignoringFields(DatastoreV3Pb.GetResponse.IN_ORDER_FIELD_NUMBER) + .isEqualTo(remoteRpcGetResponse1.build()); + + // Now do a Put. + EntityProto.Builder mutatedClientEntity2 = REFERENCE_HOLDER_2.clientEntity.toBuilder(); + EntityProto.Builder mutatedTargetEntity2 = REFERENCE_HOLDER_2.targetEntity.toBuilder(); + addProperty(mutatedClientEntity2, "newprop", 999); + addProperty(mutatedTargetEntity2, "newprop", 999); + + // One Entity in the request won't have an id yet. + EntityProto.Builder clientEntityWithNoKey6 = REFERENCE_HOLDER_6.clientEntity.toBuilder(); + EntityProto.Builder targetEntityWithNoKey6 = REFERENCE_HOLDER_6.targetEntity.toBuilder(); + clientEntityWithNoKey6 + .getKeyBuilder() + .getPathBuilder() + .getElementBuilder(0) + .clearName() + .clearId(); + targetEntityWithNoKey6 + .getKeyBuilder() + .getPathBuilder() + .getElementBuilder(0) + .clearName() + .clearId(); + + DatastoreV3Pb.PutRequest putRequest = + DatastoreV3Pb.PutRequest.newBuilder() + .setTransaction(transaction) + .addEntity(mutatedClientEntity2) + .addEntity(clientEntityWithNoKey6) + .build(); + + // It will need to send a Remote Rpc to get an id for this entity. + Reference allocatedKey = expectAllocateIds(targetEntityWithNoKey6.build()).get(0); + + // Returns the target version of the key for the Entity that already had it, and the newly + // allocated key for the other. + DatastoreV3Pb.PutResponse actualPutResp = invokePut(putRequest); + assertThat(actualPutResp.getKeyList()) + .containsExactly(mutatedTargetEntity2.getKey(), allocatedKey) + .inOrder(); + + // Issue another get request. It should use the cached results from the first. It should not + // reflect the change from the Put. + DatastoreV3Pb.GetRequest.Builder getRequest2 = DatastoreV3Pb.GetRequest.newBuilder(); + getRequest2.setTransaction(transaction); + getRequest2.addKey(REFERENCE_HOLDER_1.clientReference); + getRequest2.addKey(REFERENCE_HOLDER_2.clientReference); + getRequest2.addKey(REFERENCE_HOLDER_3.clientReference); + getRequest2.addKey(REFERENCE_HOLDER_4.targetReference); // This key already has the target app. + getRequest2.addKey(REFERENCE_HOLDER_5.clientReference); + + // It doesn't send the request for 1 and 2, because they're already in the cache. + DatastoreV3Pb.GetRequest rewrittenGetRequest2 = + DatastoreV3Pb.GetRequest.newBuilder() + .addKey(REFERENCE_HOLDER_3.targetReference) + .addKey(REFERENCE_HOLDER_4.targetReference) + .addKey(REFERENCE_HOLDER_5.targetReference) + .build(); + + // A mix of Entities, Missing, and Deferred. + DatastoreV3Pb.GetResponse.Builder remoteRpcGetResponse2 = + DatastoreV3Pb.GetResponse.newBuilder(); + remoteRpcGetResponse2.addEntityBuilder().setEntity(REFERENCE_HOLDER_3.targetEntity); + remoteRpcGetResponse2.addDeferred(REFERENCE_HOLDER_4.targetReference); + remoteRpcGetResponse2.addEntityBuilder().setKey(REFERENCE_HOLDER_5.targetReference); + + // Merges both cached data and data from the second GetResponse. + DatastoreV3Pb.GetResponse.Builder expectedGetResponse2 = + DatastoreV3Pb.GetResponse.newBuilder() + .setInOrder(false); // Because there is a deferred result. + expectedGetResponse2.addEntityBuilder().setKey(REFERENCE_HOLDER_1.targetReference); + expectedGetResponse2 + .addEntityBuilder() + .setEntity(REFERENCE_HOLDER_2.targetEntity); // Not from Put. + expectedGetResponse2.addEntityBuilder().setEntity(REFERENCE_HOLDER_3.targetEntity); + expectedGetResponse2.addDeferred(REFERENCE_HOLDER_4.targetReference); + expectedGetResponse2.addEntityBuilder().setKey(REFERENCE_HOLDER_5.targetReference); + + expectRemoteRpcGet(rewrittenGetRequest2, remoteRpcGetResponse2.build()); + + DatastoreV3Pb.GetResponse actualGetResp2 = invokeGet(getRequest2.build()); + assertThat(actualGetResp2).ignoringRepeatedFieldOrder().isEqualTo(expectedGetResponse2.build()); + } + + @Test + public void testQueriesRewriteReferences() { + OnestoreEntity.PropertyValue.ReferenceValue targetRefValue = + OnestoreEntity.PropertyValue.ReferenceValue.newBuilder().setApp(TARGET_APP_ID).build(); + + DatastoreV3Pb.Query.Builder query = DatastoreV3Pb.Query.newBuilder().setApp(TARGET_APP_ID); + query + .addFilterBuilder() + .addPropertyBuilder() + .setName("name") + .setMultiple(false) + .setValue(OnestoreEntity.PropertyValue.newBuilder().setReferenceValue(targetRefValue)); + assertFalse(RemoteDatastore.rewriteQueryAppIds(query, TARGET_APP_ID)); + OnestoreEntity.PropertyValue.ReferenceValue clientRefValue = + OnestoreEntity.PropertyValue.ReferenceValue.newBuilder().setApp(CLIENT_APP_ID).build(); + query = DatastoreV3Pb.Query.newBuilder(); + query.setApp(CLIENT_APP_ID); + query + .addFilterBuilder() + .addPropertyBuilder() + .setName("name") + .setMultiple(false) + .setValue(OnestoreEntity.PropertyValue.newBuilder().setReferenceValue(clientRefValue)); + + // check that a non-reference property is ignored + query + .addFilterBuilder() + .addPropertyBuilder() + .setName("name") + .setMultiple(false) + .setValue( + OnestoreEntity.PropertyValue.newBuilder() + .setStringValue(ByteString.copyFromUtf8("A string"))); + + assertTrue(RemoteDatastore.rewriteQueryAppIds(query, TARGET_APP_ID)); + + assertEquals(TARGET_APP_ID, query.getApp()); + assertEquals( + TARGET_APP_ID, query.getFilter(0).getProperty(0).getValue().getReferenceValue().getApp()); + assertWithMessage("string shouldn't be a reference value") + .that(query.getFilter(1).getProperty(0).getValue().hasReferenceValue()) + .isFalse(); + } + + @Test + public void rewritesPutAppIds() { + OnestoreEntity.PropertyValue.ReferenceValue targetRefValue = + OnestoreEntity.PropertyValue.ReferenceValue.newBuilder() + .setApp(TARGET_APP_ID) + .buildPartial(); + + DatastoreV3Pb.PutRequest.Builder put = DatastoreV3Pb.PutRequest.newBuilder(); + OnestoreEntity.EntityProto.Builder entity = put.addEntityBuilder(); + entity.getKeyBuilder().setApp(TARGET_APP_ID); + entity + .addPropertyBuilder() + .setName("name") + .setMultiple(false) + .setValue(OnestoreEntity.PropertyValue.newBuilder().setReferenceValue(targetRefValue)); + assertFalse(RemoteDatastore.rewritePutAppIds(put, TARGET_APP_ID)); + + OnestoreEntity.PropertyValue.ReferenceValue clientRefValue = + OnestoreEntity.PropertyValue.ReferenceValue.newBuilder() + .setApp(CLIENT_APP_ID) + .buildPartial(); + entity = put.addEntityBuilder(); + entity.getKeyBuilder().setApp(CLIENT_APP_ID); + entity + .addPropertyBuilder() + .setName("name") + .setMultiple(false) + .setValue(OnestoreEntity.PropertyValue.newBuilder().setReferenceValue(clientRefValue)); + + // check that a non-reference property is ignored + entity + .addPropertyBuilder() + .setName("name") + .setMultiple(false) + .setValue( + OnestoreEntity.PropertyValue.newBuilder() + .setStringValue(ByteString.copyFromUtf8("a string"))); + + assertTrue(RemoteDatastore.rewritePutAppIds(put, TARGET_APP_ID)); + + assertEquals(TARGET_APP_ID, entity.getKey().getApp()); + assertEquals(TARGET_APP_ID, entity.getProperty(0).getValue().getReferenceValue().getApp()); + + assertWithMessage("string shouldn't be a reference value") + .that(entity.getProperty(1).getValue().hasReferenceValue()) + .isFalse(); + } + + @Test + public void rewritesAncestorApp() { + OnestoreEntity.Reference.Builder ancestor = + OnestoreEntity.Reference.newBuilder() + .setApp(CLIENT_APP_ID) + .setPath( + OnestoreEntity.Path.newBuilder() + .addElement( + OnestoreEntity.Path.Element.newBuilder().setType("type").setName("name"))); + DatastoreV3Pb.Query.Builder query = + DatastoreV3Pb.Query.newBuilder().setApp(TARGET_APP_ID).setAncestor(ancestor); + assertTrue(RemoteDatastore.rewriteQueryAppIds(query, TARGET_APP_ID)); + ancestor.setApp(TARGET_APP_ID); + query.setAncestor(ancestor); + assertFalse(RemoteDatastore.rewriteQueryAppIds(query, TARGET_APP_ID)); + } + + private static EntityProto createEntityProto(String appId, int index) { + OnestoreEntity.Reference.Builder key = OnestoreEntity.Reference.newBuilder().setApp(appId); + key.getPathBuilder() + .addElement( + OnestoreEntity.Path.Element.newBuilder().setType("somekind").setName("name " + index)); + + EntityProto.Builder entity = EntityProto.newBuilder().setKey(key); + + // TODO: There are utilities for this, but they are all under + // apphosting/datastore/testing which we may not want to depend on from here. + OnestoreEntity.Path group = + OnestoreEntity.Path.newBuilder().addElement(key.getPath().getElement(0)).build(); + entity.setEntityGroup(group); + + addProperty(entity, "someproperty", index); + + return entity.buildPartial(); + } + + private static void addProperty(EntityProto.Builder entity, String propertyName, int value) { + OnestoreEntity.Property property = + OnestoreEntity.Property.newBuilder() + .setName(propertyName) + .setMultiple(false) + .setValue(OnestoreEntity.PropertyValue.newBuilder().setInt64Value(value)) + .build(); + entity.addProperty(property); + } + + /** + * Sets mock expectations for allocating new ids for the given Entities. The RemoteApi + * accomplishes this by calling a special method: remote_datastore.GetIDs. Note that it does not + * actually call allocateIds. + * + * @param entities the entities that need ids allocated. + * @return the References that are mocked to be returned. + */ + private List expectAllocateIds(EntityProto... entities) { + DatastoreV3Pb.PutRequest.Builder expectedReq = DatastoreV3Pb.PutRequest.newBuilder(); + DatastoreV3Pb.PutResponse.Builder resp = DatastoreV3Pb.PutResponse.newBuilder(); + + List allocatedKeys = Lists.newLinkedList(); + int idSeed = 444; + for (OnestoreEntity.EntityProto entity : entities) { + // This is copied from the impl. It sends over empty entities. The key should have a kind, + // but no id/name yet. + OnestoreEntity.EntityProto.Builder reqEntity = expectedReq.addEntityBuilder(); + reqEntity.getKeyBuilder().mergeFrom(entity.getKey()); + reqEntity.getEntityGroupBuilder(); + + // The response will have an id. + Reference.Builder respKey = reqEntity.getKeyBuilder().clone(); + respKey.getPathBuilder().getElementBuilder(0).setId(idSeed); + resp.addKey(respKey); + allocatedKeys.add(respKey.build()); + + idSeed++; + } + + when(mockRemoteRpc.call( + eq(RemoteDatastore.REMOTE_API_SERVICE), + eq("GetIDs"), + eq(""), + eq(expectedReq.build().toByteArray()))) + .thenReturn(resp.build().toByteArray()); + + return allocatedKeys; + } + + private DatastoreV3Pb.Transaction beginTransaction() throws InvalidProtocolBufferException { + DatastoreV3Pb.BeginTransactionRequest beginTxnRequest = + DatastoreV3Pb.BeginTransactionRequest.newBuilder().setApp(CLIENT_APP_ID).build(); + + byte[] txBytes = + remoteDatastore.handleDatastoreCall("BeginTransaction", beginTxnRequest.toByteArray()); + return DatastoreV3Pb.Transaction.parseFrom(txBytes, ExtensionRegistry.getEmptyRegistry()); + } + + private DatastoreV3Pb.GetResponse invokeGet(DatastoreV3Pb.GetRequest req) + throws InvalidProtocolBufferException { + byte[] actualByteResponse = remoteDatastore.handleDatastoreCall("Get", req.toByteArray()); + + return DatastoreV3Pb.GetResponse.parseFrom( + actualByteResponse, ExtensionRegistry.getEmptyRegistry()); + } + + private DatastoreV3Pb.PutResponse invokePut(DatastoreV3Pb.PutRequest req) + throws InvalidProtocolBufferException { + byte[] actualByteResponse = remoteDatastore.handleDatastoreCall("Put", req.toByteArray()); + + return DatastoreV3Pb.PutResponse.parseFrom( + actualByteResponse, ExtensionRegistry.getEmptyRegistry()); + } + + private void expectRemoteRpcGet( + DatastoreV3Pb.GetRequest expectedReq, DatastoreV3Pb.GetResponse resp) { + when(mockRemoteRpc.call( + eq(RemoteDatastore.DATASTORE_SERVICE), + eq("Get"), + eq(""), + eq(expectedReq.toByteArray()))) + .thenReturn(resp.toByteArray()); + } +} diff --git a/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/RemoteRpcTest.java b/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/RemoteRpcTest.java new file mode 100644 index 000000000..9f5cece7f --- /dev/null +++ b/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/RemoteRpcTest.java @@ -0,0 +1,105 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.tools.remoteapi; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth.assertWithMessage; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.google.apphosting.base.protos.api_bytes.RemoteApiPb; +import com.google.protobuf.ExtensionRegistry; +import com.google.protobuf.InvalidProtocolBufferException; +import java.io.IOException; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +/** + * Test for {@link RemoteRpc}. + * + */ +@RunWith(JUnit4.class) +public class RemoteRpcTest { + + @Rule public final MockitoRule mocks = MockitoJUnit.rule(); + + @Mock private AppEngineClient appEngineClient; + + /** + * Test that every RPC has a request id and that these ids differ. + */ + @Test + public void testRequestId() throws IOException { + RemoteRpc remoteRpc = new RemoteRpc(appEngineClient); + byte[] dummyRequest = {1, 2, 3, 4}; + RemoteApiPb.Response responseProto = RemoteApiPb.Response.getDefaultInstance(); + AppEngineClient.Response dummyResponse = + new AppEngineClient.Response(200, responseProto.toByteArray(), UTF_8); + when(appEngineClient.getRemoteApiPath()).thenReturn("/path/one"); + when(appEngineClient + .post(eq("/path/one"), eq("application/octet-stream"), any(byte[].class))) + .thenReturn(dummyResponse); + + ArgumentCaptor byteArrayCaptor = ArgumentCaptor.forClass(byte[].class); + + remoteRpc.call("foo", "bar", "logSuffix", dummyRequest); + + verify(appEngineClient) + .post(eq("/path/one"), eq("application/octet-stream"), byteArrayCaptor.capture()); + RemoteApiPb.Request request1 = parseRequest(byteArrayCaptor.getValue()); + assertThat(request1.getServiceName()).isEqualTo("foo"); + assertThat(request1.getMethod()).isEqualTo("bar"); + assertThat(request1.getRequest().toByteArray()).isEqualTo(dummyRequest); + String id1 = request1.getRequestId(); + + when(appEngineClient.getRemoteApiPath()).thenReturn("/path/two"); + when(appEngineClient + .post(eq("/path/two"), eq("application/octet-stream"), any(byte[].class))) + .thenReturn(dummyResponse); + + remoteRpc.call("foo", "bar", "logSuffix", dummyRequest); + + verify(appEngineClient) + .post(eq("/path/two"), eq("application/octet-stream"), byteArrayCaptor.capture()); + RemoteApiPb.Request request2 = parseRequest(byteArrayCaptor.getValue()); + assertThat(request2.getServiceName()).isEqualTo("foo"); + assertThat(request2.getMethod()).isEqualTo("bar"); + assertThat(request2.getRequest().toByteArray()).isEqualTo(dummyRequest); + String id2 = request2.getRequestId(); + assertWithMessage("Expected '%s' != '%s[", id1, id2).that(id1.equals(id2)).isFalse(); + } + + private static RemoteApiPb.Request parseRequest(byte[] bytes) { + RemoteApiPb.Request.Builder parsedRequest = RemoteApiPb.Request.newBuilder(); + try { + parsedRequest.mergeFrom(bytes, ExtensionRegistry.getEmptyRegistry()); + } catch (InvalidProtocolBufferException e) { + throw new RuntimeException(e); + } + // assertThat(parsedRequest.mergeFrom(bytes)).isTrue(); + return parsedRequest.build(); + } +} diff --git a/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/ThreadLocalDelegateTest.java b/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/ThreadLocalDelegateTest.java new file mode 100644 index 000000000..7b3967cbc --- /dev/null +++ b/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/ThreadLocalDelegateTest.java @@ -0,0 +1,72 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.tools.remoteapi; + +import static com.google.common.truth.Truth.assertThat; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.google.apphosting.api.ApiProxy; +import com.google.apphosting.api.ApiProxy.Delegate; +import com.google.apphosting.api.ApiProxy.Environment; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeoutException; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +/** + */ +@RunWith(JUnit4.class) +public class ThreadLocalDelegateTest { + + @Rule public final MockitoRule mocks = MockitoJUnit.rule(); + + @Mock private Delegate global; + @Mock private Delegate local; + + @Test + public void testSetDelegateForThread() + throws ExecutionException, TimeoutException, InterruptedException { + when(local.getRequestThreads(null)).thenReturn(ImmutableList.of()); + when(global.getRequestThreads(null)).thenReturn(Lists.newArrayList(null, null)); + + ApiProxy.setDelegate(new ThreadLocalDelegate<>(global, local)); + + Delegate delegate = ApiProxy.getDelegate(); + + assertThat(delegate.getRequestThreads(null)).isEmpty(); + + Executors.newSingleThreadExecutor() + .submit(() -> { + Delegate delegate1 = ApiProxy.getDelegate(); + assertThat(delegate1.getRequestThreads(null)).hasSize(2); + }) + .get(1, SECONDS); + + verify(local).getRequestThreads(null); + verify(global).getRequestThreads(null); + } +} diff --git a/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/testing/RemoteApiSharedTests.java b/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/testing/RemoteApiSharedTests.java new file mode 100644 index 000000000..90d7c1c6e --- /dev/null +++ b/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/testing/RemoteApiSharedTests.java @@ -0,0 +1,652 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.tools.remoteapi.testing; + +import com.google.appengine.api.datastore.DatastoreService; +import com.google.appengine.api.datastore.DatastoreServiceFactory; +import com.google.appengine.api.datastore.Entity; +import com.google.appengine.api.datastore.EntityNotFoundException; +import com.google.appengine.api.datastore.Key; +import com.google.appengine.api.datastore.KeyFactory; +import com.google.appengine.api.datastore.Query; +import com.google.appengine.api.datastore.Transaction; +import com.google.appengine.api.datastore.TransactionOptions; +import com.google.appengine.tools.remoteapi.RemoteApiInstaller; +import com.google.appengine.tools.remoteapi.RemoteApiOptions; +import com.google.common.base.Supplier; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.LinkedListMultimap; +import com.google.common.collect.Multimap; +import com.google.errorprone.annotations.CanIgnoreReturnValue; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.logging.Logger; + +/** + * Carries out a series of tests to exercise the Remote API. This can be invoked from within an App + * Engine environment or outside of one. It can target a Java or Python app. + * + */ +public class RemoteApiSharedTests { + + private static final Logger logger = Logger.getLogger(RemoteApiSharedTests.class.getName()); + + private final String localAppId; + private final String remoteAppId; + private final String username; + private final String password; + private final String server; + private final String remoteApiPath; + private final int port; + private final boolean testKeysCreatedBeforeRemoteApiInstall; + private final boolean expectRemoteAppIdsOnKeysAfterInstallingRemoteApi; + + /** + * Builder for a {@link RemoteApiSharedTests} with some sensible defaults. + */ + public static class Builder { + private String localAppId; + private String remoteAppId; + private String username; + private String password; + private String server; + + /** This is the default for Java. Needs to be /_ah/remote_api for Python. */ + private String remoteApiPath = "/remote_api"; + + /** Default for apps running in the App Engine environment. */ + private int port = 443; + + /** + * This should be set to false for Non-Hosted clients. They won't have an Environment available + * to generate Keys until the Remote API is installed. + */ + private boolean testKeysCreatedBeforeRemoteApiInstall = true; + + /** + * Allow these tests to be turned off because they rely on recent changes that haven't been + * rolled out everywhere yet. Targeting 1.8.8. + * See http://b/11254141 and http://b/10788115 + * TODO: Remove this. + */ + private boolean expectRemoteAppIdsOnKeysAfterInstallingRemoteApi = true; + + @CanIgnoreReturnValue + public Builder setLocalAppId(String localAppId) { + this.localAppId = localAppId; + return this; + } + + @CanIgnoreReturnValue + public Builder setRemoteAppId(String remoteAppId) { + this.remoteAppId = remoteAppId; + return this; + } + + @CanIgnoreReturnValue + public Builder setUsername(String username) { + this.username = username; + return this; + } + + @CanIgnoreReturnValue + public Builder setPassword(String password) { + this.password = password; + return this; + } + + @CanIgnoreReturnValue + public Builder setServer(String server) { + this.server = server; + return this; + } + + @CanIgnoreReturnValue + public Builder setPort(int port) { + this.port = port; + return this; + } + + public RemoteApiSharedTests build() { + return new RemoteApiSharedTests( + localAppId, + remoteAppId, + username, + password, + server, + remoteApiPath, + port, + testKeysCreatedBeforeRemoteApiInstall, + expectRemoteAppIdsOnKeysAfterInstallingRemoteApi); + } + } + + private RemoteApiSharedTests( + String localAppId, + String remoteAppId, + String username, + String password, + String server, + String remoteApiPath, + int port, + boolean testKeysCreatedBeforeRemoteApiInstall, + boolean expectRemoteAppIdsOnKeysAfterInstallingRemoteApi) { + this.localAppId = localAppId; + this.remoteAppId = remoteAppId; + this.username = username; + this.password = password; + this.server = server; + this.remoteApiPath = remoteApiPath; + this.port = port; + this.testKeysCreatedBeforeRemoteApiInstall = testKeysCreatedBeforeRemoteApiInstall; + this.expectRemoteAppIdsOnKeysAfterInstallingRemoteApi = + expectRemoteAppIdsOnKeysAfterInstallingRemoteApi; + } + + /** + * Throws an exception if any errors are encountered. If it completes successfully, then all test + * cases passed. + */ + public void runTests() throws IOException { + RemoteApiOptions options = new RemoteApiOptions() + .server(server, port) + .credentials(username, password) + .remoteApiPath(remoteApiPath); + + // Once we install the RemoteApi, all keys will start using the remote app id. We'll store some + // keys with the local app id first. + LocalKeysHolder localKeysHolder = null; + LocalEntitiesHolder localEntitiesHolder = null; + if (testKeysCreatedBeforeRemoteApiInstall) { + localKeysHolder = new LocalKeysHolder(); + localEntitiesHolder = new LocalEntitiesHolder(); + } + + RemoteApiInstaller installer = new RemoteApiInstaller(); + installer.install(options); + + // Update the options with reusable credentials. + options.reuseCredentials(username, installer.serializeCredentials()); + // Execute our tests using the initial installation. + try { + doTest(localKeysHolder, localEntitiesHolder); + } finally { + installer.uninstall(); + } + + if (testKeysCreatedBeforeRemoteApiInstall) { + // Make sure uninstalling brings the keys back to the local app id. + assertNewKeysUseLocalAppId(); + } + + installer.install(options); + // Execute our tests using the second installation. + try { + doTest(localKeysHolder, localEntitiesHolder); + } finally { + installer.uninstall(); + } + } + + /** + * Runs a series of tests using keys with both local app ids and remote app ids. + */ + private void doTest(LocalKeysHolder localKeysHolder, LocalEntitiesHolder localEntitiesHolder) { + DatastoreService ds = DatastoreServiceFactory.getDatastoreService(); + + List tests = ImmutableList.of( + new PutAndGetTester(), + new PutAndGetInTransactionTester(), + new QueryTester(), + new DeleteTester(), + new XgTransactionTester()); + + // Run each test once with local keys and once with remote keys. + for (RemoteApiUnitTest test : tests) { + if (localKeysHolder != null) { + test.run( + ds, + localKeysHolder.createSupplierForFreshKind(), + localEntitiesHolder.createSupplierForFreshKind()); + logger.info("Test passed with local keys: " + test.getClass().getName()); + } + + test.run(ds, new RemoteKeySupplier(), new RemoteEntitySupplier()); + logger.info("Test passed with remote keys: " + test.getClass().getName()); + } + } + + private class PutAndGetTester implements RemoteApiUnitTest { + + @Override + public void run( + DatastoreService ds, Supplier keySupplier, Supplier entitySupplier) { + Entity entity1 = new Entity(keySupplier.get()); + entity1.setProperty("prop1", 75L); + + // Verify results of Put + Key keyReturnedFromPut = ds.put(entity1); + assertEquals(entity1.getKey(), keyReturnedFromPut); + + // Make sure we can retrieve it again. + assertGetEquals(ds, keyReturnedFromPut, entity1); + + // Test EntityNotFoundException + Key unsavedKey = keySupplier.get(); + assertEntityNotFoundException(ds, unsavedKey); + + // Test batch get + Entity entity2 = new Entity(keySupplier.get()); + entity2.setProperty("prop1", 88L); + ds.put(entity2); + + Map batchGetResult = + ds.get(Arrays.asList(entity1.getKey(), unsavedKey, entity2.getKey())); + + // Omits the unsaved key from results. + assertEquals(2, batchGetResult.size()); + assertEquals(entity1, batchGetResult.get(entity1.getKey())); + assertEquals(entity2, batchGetResult.get(entity2.getKey())); + + // Test Put and Get with id generated by Datastore backend. + Entity entity3 = entitySupplier.get(); + entity3.setProperty("prop1", 35L); + assertNoIdOrName(entity3.getKey()); + + Key assignedKey = ds.put(entity3); + + assertTrue(assignedKey.getId() > 0); + assertEquals(assignedKey, entity3.getKey()); + assertGetEquals(ds, assignedKey, entity3); + } + } + + private class PutAndGetInTransactionTester implements RemoteApiUnitTest { + + @Override + public void run( + DatastoreService ds, Supplier keySupplier, Supplier entitySupplier) { + // Put a fresh entity. + Entity originalEntity = new Entity(getFreshKindName()); + originalEntity.setProperty("prop1", 75L); + ds.put(originalEntity); + Key key = originalEntity.getKey(); + + // Prepare a new version of it with a different property value. + Entity mutatedEntity = new Entity(key); + mutatedEntity.setProperty("prop1", 76L); + + // Test Get/Put within a transaction. + Transaction txn = ds.beginTransaction(); + assertGetEquals(ds, key, originalEntity); + ds.put(mutatedEntity); // Write the mutated Entity. + assertGetEquals(ds, key, originalEntity); // Within a txn, the put is not yet reflected. + txn.commit(); + + // Now that the txn is committed, the mutated entity will show up in Get. + assertGetEquals(ds, key, mutatedEntity); + } + } + + private class QueryTester implements RemoteApiUnitTest { + + @Override + public void run( + DatastoreService ds, Supplier keySupplier, Supplier entitySupplier) { + // Note that we can't use local keys here. Query will fail if you set an ancestor whose app + // id does not match the "global" AppIdNamespace. + // TODO: Consider making it more lenient, but it's not a big deal. Users can + // just use a Key that was created after installing the Remote API. + Entity entity = new Entity(getFreshKindName()); + entity.setProperty("prop1", 99L); + ds.put(entity); + + // Make sure we can retrieve it via a query. + Query query = new Query(entity.getKind()); + query.setAncestor(entity.getKey()); + query.setFilter( + new Query.FilterPredicate( + Entity.KEY_RESERVED_PROPERTY, + Query.FilterOperator.GREATER_THAN_OR_EQUAL, + entity.getKey())); + + Entity queryResult = ds.prepare(query).asSingleEntity(); + + // Queries return the Entities with the remote app id. + assertRemoteAppId(queryResult.getKey()); + assertEquals(99L, queryResult.getProperty("prop1")); + } + } + + private class DeleteTester implements RemoteApiUnitTest { + + @Override + public void run( + DatastoreService ds, Supplier keySupplier, Supplier entitySupplier) { + Key key = keySupplier.get(); + Entity entity = new Entity(key); + entity.setProperty("prop1", 75L); + ds.put(entity); + + assertGetEquals(ds, key, entity); + ds.delete(key); + assertEntityNotFoundException(ds, key); + } + } + + private static class XgTransactionTester implements RemoteApiUnitTest { + + @Override + public void run( + DatastoreService ds, Supplier keySupplier, Supplier entitySupplier) { + Transaction txn = ds.beginTransaction(TransactionOptions.Builder.withXG(true)); + if (ds.put(new Entity("xgfoo")).getId() == 0) { + throw new RuntimeException("first entity should have received an id"); + } + if (ds.put(new Entity("xgfoo")).getId() == 0) { + throw new RuntimeException("second entity should have received an id"); + } + txn.commit(); + } + } + + /** + * Simple interface for test cases. + */ + private interface RemoteApiUnitTest { + + /** + * Runs the test case using the given DatastoreService and Keys from the given KeySupplier. + * + * @throws RuntimeException if the test fails. + */ + void run(DatastoreService ds, Supplier keySupplier, Supplier entitySupplier); + } + + /** + * A {@link Supplier} that creates Keys with the Remote app id. Assumes the Remote API is already + * installed. + */ + private class RemoteKeySupplier implements Supplier { + + private final String kind; + private int nameCounter = 0; + + private RemoteKeySupplier() { + this.kind = getFreshKindName(); + } + + @Override + public Key get() { + // This assumes that the remote api has already been installed. + Key key = KeyFactory.createKey(kind, "somename" + nameCounter); + if (expectRemoteAppIdsOnKeysAfterInstallingRemoteApi) { + assertRemoteAppId(key); + } + nameCounter++; + + return key; + } + } + + /** + * A {@link Supplier} that creates Entities with a Key with the Remote app id (and no id or name + * set.) Assumes the Remote API is already installed. + */ + private class RemoteEntitySupplier implements Supplier { + + private final String kind; + + private RemoteEntitySupplier() { + this.kind = getFreshKindName(); + } + + @Override + public Entity get() { + // This assumes that the remote api has already been installed. + Entity entity = new Entity(kind); + if (expectRemoteAppIdsOnKeysAfterInstallingRemoteApi) { + assertRemoteAppId(entity.getKey()); + } + + return entity; + } + } + + /** + * Creates and caches Keys with the local app id. Assumes that the Remote API has not yet been + * installed when a new instance is created. + */ + private class LocalKeysHolder { + private static final int NUM_KINDS = 50; + private static final int NUM_KEYS_PER_KIND = 20; + + private Multimap keysByKind; + + public LocalKeysHolder() { + keysByKind = LinkedListMultimap.create(); + + for (int kindCounter = 0; kindCounter < NUM_KINDS; ++kindCounter) { + String kind = getFreshKindName(); + for (int keyNameCounter = 0; keyNameCounter < NUM_KEYS_PER_KIND; ++keyNameCounter) { + String name = "somename" + keyNameCounter; + + Key key = KeyFactory.createKey(kind, name); + assertLocalAppId(key); + keysByKind.put(kind, key); + } + } + } + + private Supplier createSupplierForFreshKind() { + String kind = keysByKind.keySet().iterator().next(); + final Iterator keysIterator = keysByKind.get(kind).iterator(); + keysByKind.removeAll(kind); + + return new Supplier() { + @Override + public Key get() { + return keysIterator.next(); + } + }; + } + } + + /** + * Creates and caches Entities with Keys containing the local app id (but no id or name set.) + * Assumes that the Remote API has not yet been installed when a new instance is created. + */ + private class LocalEntitiesHolder { + private static final int NUM_KINDS = 50; + private static final int NUM_ENTITIES_PER_KIND = 20; + + private Multimap entitiesByKind; + + public LocalEntitiesHolder() { + entitiesByKind = LinkedListMultimap.create(); + + for (int kindCounter = 0; kindCounter < NUM_KINDS; ++kindCounter) { + String kind = getFreshKindName(); + for (int i = 0; i < NUM_ENTITIES_PER_KIND; ++i) { + // Will get a default Key with the local app id and no id or name. + Entity entity = new Entity(kind); + + assertLocalAppId(entity.getKey()); + entitiesByKind.put(kind, entity); + } + } + } + + private Supplier createSupplierForFreshKind() { + String kind = entitiesByKind.keySet().iterator().next(); + final Iterator entitiesIterator = entitiesByKind.get(kind).iterator(); + entitiesByKind.removeAll(kind); + + return new Supplier() { + @Override + public Entity get() { + return entitiesIterator.next(); + } + }; + } + } + + private void assertTrue(boolean condition, String message) { + if (!condition) { + throw new RuntimeException(message); + } + } + + private void assertTrue(boolean condition) { + assertTrue(condition, ""); + } + + private void assertEquals(Object o1, Object o2) { + assertEquals(o1, o2, "Expected " + o1 + " to equal " + o2); + } + + private void assertEquals(Object o1, Object o2, String message) { + if (o1 == null) { + assertTrue(o2 == null, message); + return; + } + assertTrue(o1.equals(o2), message); + } + + /** Special version of assertEquals for Entities that will ignore app ids on Keys. */ + private void assertEquals(Entity e1, Entity e2) { + if (e1 == null) { + assertTrue(e2 == null); + return; + } + + assertEquals(e1.getProperties(), e2.getProperties()); + assertEquals(e1.getKey(), e2.getKey()); + } + + /** Special version of assertEquals for Keys that will ignore app ids. */ + private void assertEquals(Key k1, Key k2) { + if (k1 == null) { + assertTrue(k2 == null); + return; + } + + assertEquals(k1.getKind(), k2.getKind()); + assertEquals(k1.getId(), k2.getId()); + assertEquals(k1.getName(), k2.getName()); + + assertEquals(k1.getParent(), k2.getParent()); + } + + private void assertLocalAppId(Key key) { + assertAppIdsMatchIgnoringPartition(localAppId, key.getAppId()); + } + + private void assertRemoteAppId(Key key) { + assertAppIdsMatchIgnoringPartition(remoteAppId, key.getAppId()); + } + + /** + * The e2e testing framework is not very strict about requiring fully specified app ids. + * Therefore, we might get "display" app ids given to us and we need to consider "s~foo" and "foo" + * to be equal. + */ + private void assertAppIdsMatchIgnoringPartition(String appId1, String appId2) { + if (appId1.equals(appId2)) { + // Exact match. + return; + } + + // Consider s~foo == foo. + assertEquals( + stripPartitionFromAppId(appId1), + stripPartitionFromAppId(appId2), + "Expected app id to be: " + appId1 + ", but was: " + appId2); + } + + /** + * Example conversions: + * + * foo => foo + * s~foo => foo + * e~foo => foo + * hrd~foo => foo (Doesn't exist in App Engine today, but this code will support partitions + * greater than 1 char.) + */ + private String stripPartitionFromAppId(String appId) { + int partitionIndex = appId.indexOf('~'); + if (partitionIndex != -1 && appId.length() > partitionIndex + 1) { + return appId.substring(partitionIndex + 1); + } + return appId; + } + + private void assertNewKeysUseLocalAppId() { + assertLocalAppId(KeyFactory.createKey(getFreshKindName(), "somename")); + } + + private void assertNoIdOrName(Key key) { + assertEquals(0L, key.getId()); + assertEquals(null, key.getName()); + } + + private void assertGetEquals(DatastoreService ds, Key keyToGet, Entity expectedEntity) { + // Test the single key api. + Entity entityFromGet = quietGet(ds, keyToGet); + assertEquals(expectedEntity, entityFromGet); + assertRemoteAppId(entityFromGet.getKey()); + + // Test the multi-get api. + Map getResults = ds.get(Collections.singletonList(keyToGet)); + assertEquals(1, getResults.size()); + Entity entityFromBatchGet = getResults.get(keyToGet); + assertEquals(expectedEntity, entityFromBatchGet); + assertRemoteAppId(entityFromBatchGet.getKey()); + } + + private void assertEntityNotFoundException(DatastoreService ds, Key missingKey) { + try { + ds.get(missingKey); + throw new RuntimeException("Did not receive expected exception"); + } catch (EntityNotFoundException e) { + // expected + } + } + + /** + * Propagates {@link EntityNotFoundException} as {@link RuntimeException} + */ + private Entity quietGet(DatastoreService ds, Key key) { + try { + return ds.get(key); + } catch (EntityNotFoundException e) { + throw new RuntimeException(e); + } + } + + /** + * Generates a Kind name that has not yet been used. + */ + private String getFreshKindName() { + return "testkind" + UUID.randomUUID(); + } +} diff --git a/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/testing/StubCredential.java b/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/testing/StubCredential.java new file mode 100644 index 000000000..b40d35ef3 --- /dev/null +++ b/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/testing/StubCredential.java @@ -0,0 +1,40 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.tools.remoteapi.testing; + +import com.google.api.client.auth.oauth2.Credential; +import com.google.api.client.http.HttpRequest; +import java.io.IOException; + +/** + * A stub {@link Credential} for testing. + */ +public class StubCredential extends Credential { + + public StubCredential() { + super(new AccessMethod() { + @Override + public String getAccessTokenFromRequest(HttpRequest request) { + return null; + } + + @Override + public void intercept(HttpRequest request, String accessToken) throws IOException { + } + }); + } +} diff --git a/remoteapi/src/test/resources/com/google/appengine/tools/remoteapi/testdata/test.pkcs12 b/remoteapi/src/test/resources/com/google/appengine/tools/remoteapi/testdata/test.pkcs12 new file mode 100644 index 000000000..51c07e509 Binary files /dev/null and b/remoteapi/src/test/resources/com/google/appengine/tools/remoteapi/testdata/test.pkcs12 differ diff --git a/runtime/runtime_impl_jetty9/src/main/java/com/google/apphosting/runtime/jetty9/AppInfoFactory.java b/runtime/impl/src/main/java/com/google/apphosting/runtime/AppInfoFactory.java similarity index 87% rename from runtime/runtime_impl_jetty9/src/main/java/com/google/apphosting/runtime/jetty9/AppInfoFactory.java rename to runtime/impl/src/main/java/com/google/apphosting/runtime/AppInfoFactory.java index 3f6c16283..71cb2532b 100644 --- a/runtime/runtime_impl_jetty9/src/main/java/com/google/apphosting/runtime/jetty9/AppInfoFactory.java +++ b/runtime/impl/src/main/java/com/google/apphosting/runtime/AppInfoFactory.java @@ -14,10 +14,10 @@ * limitations under the License. */ -package com.google.apphosting.runtime.jetty9; +package com.google.apphosting.runtime; + import com.google.apphosting.base.protos.AppinfoPb; -import com.google.apphosting.utils.config.AppYaml; import java.io.File; import java.io.IOException; import java.nio.file.NoSuchFileException; @@ -84,17 +84,9 @@ public AppinfoPb.AppInfo getAppInfoFromFile(String applicationRoot, String fixed return getAppInfo(); } - public AppinfoPb.AppInfo getAppInfoFromAppYaml(AppYaml unused) throws IOException { - return getAppInfo(); - } - public AppinfoPb.AppInfo getAppInfo() { - final AppinfoPb.AppInfo.Builder appInfoBuilder = - AppinfoPb.AppInfo.newBuilder() - .setAppId(gaeApplication) - .setVersionId(gaeVersion) - .setRuntimeId("java8"); - return appInfoBuilder.build(); + public AppinfoPb.AppInfo getAppInfo() { + return AppinfoPb.AppInfo.newBuilder().setAppId(gaeApplication).setVersionId(gaeVersion).build(); } } diff --git a/runtime/runtime_impl_jetty9/src/test/java/com/google/apphosting/runtime/jetty9/AppInfoFactoryTest.java b/runtime/impl/src/test/java/com/google/apphosting/runtime/AppInfoFactoryTest.java similarity index 87% rename from runtime/runtime_impl_jetty9/src/test/java/com/google/apphosting/runtime/jetty9/AppInfoFactoryTest.java rename to runtime/impl/src/test/java/com/google/apphosting/runtime/AppInfoFactoryTest.java index 06f1b4436..e575cb94c 100644 --- a/runtime/runtime_impl_jetty9/src/test/java/com/google/apphosting/runtime/jetty9/AppInfoFactoryTest.java +++ b/runtime/impl/src/test/java/com/google/apphosting/runtime/AppInfoFactoryTest.java @@ -14,24 +14,18 @@ * limitations under the License. */ -package com.google.apphosting.runtime.jetty9; +package com.google.apphosting.runtime; import static com.google.common.base.StandardSystemProperty.USER_DIR; import static com.google.common.truth.Truth.assertThat; -import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertThrows; import com.google.appengine.tools.development.resource.ResourceExtractor; import com.google.apphosting.base.protos.AppinfoPb; -import com.google.apphosting.utils.config.AppYaml; import com.google.common.collect.ImmutableMap; -import java.io.File; -import java.io.FileInputStream; import java.io.IOException; -import java.io.InputStreamReader; import java.nio.file.NoSuchFileException; import java.nio.file.Path; -import java.nio.file.Paths; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -53,16 +47,15 @@ public final class AppInfoFactoryTest { @Before public void setUp() throws IOException { - Path projPath = Paths.get(temporaryFolder.newFolder(PROJECT_RESOURCE_NAME).getPath()); + Path projPath = Path.of(temporaryFolder.newFolder(PROJECT_RESOURCE_NAME).getPath()); appRoot = projPath.getParent().toString(); - fixedAppDir = Paths.get(projPath.toString(), "100.mydeployment").toString(); + fixedAppDir = Path.of(projPath.toString(), "100.mydeployment").toString(); ResourceExtractor.toFile(PROJECT_RESOURCE_NAME, projPath.toString()); } @Test public void getGaeService_nonDefault() throws Exception { - AppInfoFactory factory = - new AppInfoFactory(ImmutableMap.of("GAE_SERVICE", "mytestservice")); + AppInfoFactory factory = new AppInfoFactory(ImmutableMap.of("GAE_SERVICE", "mytestservice")); assertThat(factory.getGaeService()).isEqualTo("mytestservice"); } @@ -140,7 +133,6 @@ public void getAppInfo_fixedApplicationPath() throws Exception { assertThat(appInfo.getAppId()).isEqualTo("s~myapp"); assertThat(appInfo.getVersionId()).isEqualTo("mytestservice:100.mydeployment"); - assertThat(appInfo.getRuntimeId()).isEqualTo("java8"); } @Test @@ -157,7 +149,6 @@ public void getAppInfo_appRoot() throws Exception { assertThat(appInfo.getAppId()).isEqualTo("s~myapp"); assertThat(appInfo.getVersionId()).isEqualTo("mytestservice:100.mydeployment"); - assertThat(appInfo.getRuntimeId()).isEqualTo("java8"); } @Test @@ -179,7 +170,7 @@ public void getAppInfo_noAppYaml() throws Exception { assertThat(appInfo.getAppId()).isEqualTo("s~myapp"); assertThat(appInfo.getVersionId()).isEqualTo("mytestservice:100.mydeployment"); - assertThat(appInfo.getRuntimeId()).isEqualTo("java8"); + assertThat(appInfo.getApiVersion()).isEmpty(); } @Test @@ -208,14 +199,10 @@ public void getAppInfo_givenAppYaml() throws Exception { "GAE_APPLICATION", "s~myapp", "GOOGLE_CLOUD_PROJECT", "mytestproject")); - File appYamlFile = new File(fixedAppDir + "/WEB-INF/appengine-generated/app.yaml"); - AppYaml appYaml = AppYaml.parse(new InputStreamReader(new FileInputStream(appYamlFile), UTF_8)); - - AppinfoPb.AppInfo appInfo = factory.getAppInfoFromAppYaml(appYaml); + AppinfoPb.AppInfo appInfo = factory.getAppInfo(); assertThat(appInfo.getAppId()).isEqualTo("s~myapp"); assertThat(appInfo.getVersionId()).isEqualTo("mytestservice:100.mydeployment"); - assertThat(appInfo.getRuntimeId()).isEqualTo("java8"); } @Test @@ -233,6 +220,5 @@ public void getAppInfo_givenVersion() throws Exception { assertThat(appInfo.getAppId()).isEqualTo("s~myapp"); assertThat(appInfo.getVersionId()).isEqualTo("mytestservice:100.mydeployment"); - assertThat(appInfo.getRuntimeId()).isEqualTo("java8"); } } diff --git a/runtime/impl/src/test/resources/com/google/apphosting/runtime/jetty9/mytestproject/100.mydeployment/WEB-INF/appengine-generated/app.yaml b/runtime/impl/src/test/resources/com/google/apphosting/runtime/mytestproject/100.mydeployment/WEB-INF/appengine-generated/app.yaml similarity index 100% rename from runtime/impl/src/test/resources/com/google/apphosting/runtime/jetty9/mytestproject/100.mydeployment/WEB-INF/appengine-generated/app.yaml rename to runtime/impl/src/test/resources/com/google/apphosting/runtime/mytestproject/100.mydeployment/WEB-INF/appengine-generated/app.yaml diff --git a/runtime/lite/src/main/java/com/google/appengine/runtime/lite/AppEngineRuntime.java b/runtime/lite/src/main/java/com/google/appengine/runtime/lite/AppEngineRuntime.java index 2f9faad1e..a50953e63 100644 --- a/runtime/lite/src/main/java/com/google/appengine/runtime/lite/AppEngineRuntime.java +++ b/runtime/lite/src/main/java/com/google/appengine/runtime/lite/AppEngineRuntime.java @@ -23,13 +23,13 @@ import com.google.apphosting.base.protos.AppinfoPb; import com.google.apphosting.runtime.ApiDeadlineOracle; import com.google.apphosting.runtime.ApiProxyImpl; +import com.google.apphosting.runtime.AppInfoFactory; import com.google.apphosting.runtime.AppVersion; import com.google.apphosting.runtime.ApplicationEnvironment; import com.google.apphosting.runtime.SessionsConfig; import com.google.apphosting.runtime.anyrpc.APIHostClientInterface; import com.google.apphosting.runtime.http.HttpApiHostClientFactory; import com.google.apphosting.runtime.jetty9.AppEngineWebAppContext; -import com.google.apphosting.runtime.jetty9.AppInfoFactory; import com.google.apphosting.runtime.jetty9.AppVersionHandlerFactory; import com.google.apphosting.runtime.jetty9.JettyServerConnectorWithReusePort; import com.google.apphosting.runtime.jetty9.WebAppContextFactory; diff --git a/runtime/lite/src/main/java/com/google/appengine/runtime/lite/RequestHandler.java b/runtime/lite/src/main/java/com/google/appengine/runtime/lite/RequestHandler.java index 2c1af16d7..ff5b6501a 100644 --- a/runtime/lite/src/main/java/com/google/appengine/runtime/lite/RequestHandler.java +++ b/runtime/lite/src/main/java/com/google/appengine/runtime/lite/RequestHandler.java @@ -17,9 +17,9 @@ package com.google.appengine.runtime.lite; import com.google.apphosting.runtime.AppVersion; +import com.google.apphosting.runtime.AppInfoFactory; import com.google.apphosting.runtime.MutableUpResponse; import com.google.apphosting.runtime.anyrpc.AnyRpcServerContext; -import com.google.apphosting.runtime.jetty9.AppInfoFactory; import com.google.apphosting.runtime.jetty9.AppVersionHandlerFactory; import com.google.apphosting.runtime.jetty9.UPRequestTranslator; import com.google.common.flogger.GoogleLogger; diff --git a/runtime/lite/src/test/java/com/google/appengine/runtime/lite/AppEngineRuntimeTest.java b/runtime/lite/src/test/java/com/google/appengine/runtime/lite/AppEngineRuntimeTest.java index 58f38c6b2..e86493c2b 100644 --- a/runtime/lite/src/test/java/com/google/appengine/runtime/lite/AppEngineRuntimeTest.java +++ b/runtime/lite/src/test/java/com/google/appengine/runtime/lite/AppEngineRuntimeTest.java @@ -35,9 +35,9 @@ import com.google.apphosting.runtime.ApiProxyImpl; import com.google.apphosting.runtime.AppVersion; import com.google.apphosting.runtime.ApplicationEnvironment; +import com.google.apphosting.runtime.AppInfoFactory; import com.google.apphosting.runtime.SessionsConfig; import com.google.apphosting.runtime.http.FakeHttpApiHost; -import com.google.apphosting.runtime.jetty9.AppInfoFactory; import com.google.apphosting.testing.PortPicker; import com.google.common.base.Joiner; import com.google.common.collect.ImmutableMap; diff --git a/runtime/local_jetty121/src/main/java/com/google/appengine/tools/development/jetty/AppEngineAnnotationConfiguration.java b/runtime/local_jetty121/src/main/java/com/google/appengine/tools/development/jetty/AppEngineAnnotationConfiguration.java index aae9160e1..16b2a2edb 100644 --- a/runtime/local_jetty121/src/main/java/com/google/appengine/tools/development/jetty/AppEngineAnnotationConfiguration.java +++ b/runtime/local_jetty121/src/main/java/com/google/appengine/tools/development/jetty/AppEngineAnnotationConfiguration.java @@ -31,6 +31,9 @@ public class AppEngineAnnotationConfiguration extends AnnotationConfiguration { @Override public List getNonExcludedInitializers(WebAppContext context) throws Exception { + // TODO: remove this line when https://github.com/jetty/jetty.project/issues/14431 is resolved. + context.getMetaData().orderFragments(); + ArrayList nonExcludedInitializers = new ArrayList<>(super.getNonExcludedInitializers(context)); for (ServletContainerInitializer sci : nonExcludedInitializers) { diff --git a/runtime/local_jetty121/src/main/java/com/google/appengine/tools/development/jetty/JettyContainerService.java b/runtime/local_jetty121/src/main/java/com/google/appengine/tools/development/jetty/JettyContainerService.java index 086d4c107..ba79e877c 100644 --- a/runtime/local_jetty121/src/main/java/com/google/appengine/tools/development/jetty/JettyContainerService.java +++ b/runtime/local_jetty121/src/main/java/com/google/appengine/tools/development/jetty/JettyContainerService.java @@ -186,7 +186,11 @@ public void exitScope(ContextHandler.APIContext context, Request request) { // Set the location of deployment descriptor. This value might be null, // which is fine, it just means Jetty will look for it in the default // location (WEB-INF/web.xml). - context.setDescriptor(webXmlLocation == null ? null : webXmlLocation.getAbsolutePath()); + // Only set the descriptor if the web.xml file actually exists. + // Jetty 12 throws an IllegalArgumentException if the descriptor path is invalid. + if (webXmlLocation != null && webXmlLocation.exists()) { + context.setDescriptor(webXmlLocation.getAbsolutePath()); + } // Override the web.xml that Jetty automatically prepends to other // web.xml files. This is where the DefaultServlet is registered, diff --git a/runtime/local_jetty121_ee11/src/main/java/com/google/appengine/tools/development/jetty/ee11/AppEngineAnnotationConfiguration.java b/runtime/local_jetty121_ee11/src/main/java/com/google/appengine/tools/development/jetty/ee11/AppEngineAnnotationConfiguration.java index e921fce5c..7e6927df3 100644 --- a/runtime/local_jetty121_ee11/src/main/java/com/google/appengine/tools/development/jetty/ee11/AppEngineAnnotationConfiguration.java +++ b/runtime/local_jetty121_ee11/src/main/java/com/google/appengine/tools/development/jetty/ee11/AppEngineAnnotationConfiguration.java @@ -29,6 +29,8 @@ public class AppEngineAnnotationConfiguration extends AnnotationConfiguration { @Override protected List getNonExcludedInitializers(State state) { + // TODO: remove this line when https://github.com/jetty/jetty.project/issues/14431 is resolved. + state._context.getMetaData().orderFragments(); List initializers = super.getNonExcludedInitializers(state); for (ServletContainerInitializer sci : initializers) { diff --git a/runtime/local_jetty121_ee11/src/main/java/com/google/appengine/tools/development/jetty/ee11/JettyContainerService.java b/runtime/local_jetty121_ee11/src/main/java/com/google/appengine/tools/development/jetty/ee11/JettyContainerService.java index 5f1b33283..59290de3b 100644 --- a/runtime/local_jetty121_ee11/src/main/java/com/google/appengine/tools/development/jetty/ee11/JettyContainerService.java +++ b/runtime/local_jetty121_ee11/src/main/java/com/google/appengine/tools/development/jetty/ee11/JettyContainerService.java @@ -192,8 +192,11 @@ public void exitScope( // Set the location of deployment descriptor. This value might be null, // which is fine, it just means Jetty will look for it in the default // location (WEB-INF/web.xml). - context.setDescriptor(webXmlLocation == null ? null : webXmlLocation.getAbsolutePath()); - + // Jetty 12 throws an IllegalArgumentException if the descriptor path is invalid. + if (webXmlLocation != null && webXmlLocation.exists()) { + context.setDescriptor(webXmlLocation.getAbsolutePath()); + } + // Override the web.xml that Jetty automatically prepends to other // web.xml files. This is where the DefaultServlet is registered, // which serves static files. We override it to disable some diff --git a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/AppInfoFactory.java b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/AppInfoFactory.java deleted file mode 100644 index 576bad4c5..000000000 --- a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/AppInfoFactory.java +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.apphosting.runtime.jetty; - -import static java.nio.charset.StandardCharsets.UTF_8; - -import com.esotericsoftware.yamlbeans.YamlReader; -import com.google.apphosting.base.protos.AppinfoPb; -import com.google.apphosting.utils.config.AppYaml; -import com.google.common.flogger.GoogleLogger; -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.NoSuchFileException; -import java.util.Map; -import org.jspecify.annotations.Nullable; - -/** Builds AppinfoPb.AppInfo from the given ServletEngineAdapter.Config and environment. */ -public class AppInfoFactory { - - private static final GoogleLogger logger = GoogleLogger.forEnclosingClass(); - - private static final String DEFAULT_CLOUD_PROJECT = "testapp"; - private static final String DEFAULT_GAE_APPLICATION = "s~testapp"; - private static final String DEFAULT_GAE_SERVICE = "default"; - private static final String DEFAULT_GAE_VERSION = "1.0"; - /** Path in the WAR layout to app.yaml */ - private static final String APP_YAML_PATH = "WEB-INF/appengine-generated/app.yaml"; - - private final String gaeVersion; - private final String googleCloudProject; - private final String gaeApplication; - private final String gaeService; - private final String gaeServiceVersion; - - public AppInfoFactory(Map env) { - String version = env.getOrDefault("GAE_VERSION", DEFAULT_GAE_VERSION); - String deploymentId = env.getOrDefault("GAE_DEPLOYMENT_ID", null); - gaeServiceVersion = (deploymentId != null) ? version + "." + deploymentId : version; - gaeService = env.getOrDefault("GAE_SERVICE", DEFAULT_GAE_SERVICE); - // Prepend service if it exists, otherwise do not prepend DEFAULT (go/app-engine-ids) - gaeVersion = - DEFAULT_GAE_SERVICE.equals(this.gaeService) - ? this.gaeServiceVersion - : this.gaeService + ":" + this.gaeServiceVersion; - googleCloudProject = env.getOrDefault("GOOGLE_CLOUD_PROJECT", DEFAULT_CLOUD_PROJECT); - gaeApplication = env.getOrDefault("GAE_APPLICATION", DEFAULT_GAE_APPLICATION); - } - - public String getGaeService() { - return gaeService; - } - - public String getGaeVersion() { - return gaeVersion; - } - - public String getGaeServiceVersion() { - return gaeServiceVersion; - } - - public String getGaeApplication() { - return gaeApplication; - } - - /** Creates a AppinfoPb.AppInfo object. */ - public AppinfoPb.AppInfo getAppInfoFromFile(String applicationRoot, String fixedApplicationPath) - throws IOException { - // App should be located under /base/data/home/apps/appId/versionID or in the optional - // fixedApplicationPath parameter. - String applicationPath = - (fixedApplicationPath == null) - ? applicationRoot + "/" + googleCloudProject + "/" + gaeServiceVersion - : fixedApplicationPath; - - if (!new File(applicationPath).exists()) { - throw new NoSuchFileException("Application does not exist under: " + applicationPath); - } - @Nullable String apiVersion = null; - File appYamlFile = new File(applicationPath, APP_YAML_PATH); - try { - YamlReader reader = new YamlReader(Files.newBufferedReader(appYamlFile.toPath(), UTF_8)); - Object apiVersionObj = ((Map) reader.read()).get("api_version"); - if (apiVersionObj != null) { - apiVersion = (String) apiVersionObj; - } - } catch (NoSuchFileException ex) { - logger.atInfo().log( - "Cannot configure App Engine APIs, because the generated app.yaml file " - + "does not exist: %s", - appYamlFile.getAbsolutePath()); - } - return getAppInfoWithApiVersion(apiVersion); - } - - public AppinfoPb.AppInfo getAppInfoFromAppYaml(AppYaml appYaml) throws IOException { - return getAppInfoWithApiVersion(appYaml.getApi_version()); - } - - public AppinfoPb.AppInfo getAppInfoWithApiVersion(@Nullable String apiVersion) { - final AppinfoPb.AppInfo.Builder appInfoBuilder = - AppinfoPb.AppInfo.newBuilder() - .setAppId(gaeApplication) - .setVersionId(gaeVersion) - .setRuntimeId("java8"); - - if (apiVersion != null) { - appInfoBuilder.setApiVersion(apiVersion); - } - - return appInfoBuilder.build(); - } -} diff --git a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/JettyServletEngineAdapter.java b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/JettyServletEngineAdapter.java index 4676789d2..8dbdaedcf 100644 --- a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/JettyServletEngineAdapter.java +++ b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/JettyServletEngineAdapter.java @@ -28,6 +28,7 @@ import com.google.apphosting.base.protos.RuntimePb.UPRequest; import com.google.apphosting.base.protos.RuntimePb.UPResponse; import com.google.apphosting.runtime.AppEngineConstants; +import com.google.apphosting.runtime.AppInfoFactory; import com.google.apphosting.runtime.AppVersion; import com.google.apphosting.runtime.LocalRpcContext; import com.google.apphosting.runtime.MutableUpResponse; diff --git a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/http/JettyHttpHandler.java b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/http/JettyHttpHandler.java index 262affa18..07b6d44d4 100644 --- a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/http/JettyHttpHandler.java +++ b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/http/JettyHttpHandler.java @@ -25,26 +25,23 @@ import com.google.apphosting.base.protos.RuntimePb; import com.google.apphosting.runtime.ApiProxyImpl; import com.google.apphosting.runtime.AppEngineConstants; +import com.google.apphosting.runtime.AppInfoFactory; import com.google.apphosting.runtime.AppVersion; import com.google.apphosting.runtime.BackgroundRequestCoordinator; import com.google.apphosting.runtime.LocalRpcContext; import com.google.apphosting.runtime.RequestManager; -import com.google.apphosting.runtime.RequestRunner; import com.google.apphosting.runtime.RequestRunner.EagerRunner; import com.google.apphosting.runtime.ResponseAPIData; import com.google.apphosting.runtime.ServletEngineAdapter; import com.google.apphosting.runtime.anyrpc.AnyRpcServerContext; -import com.google.apphosting.runtime.jetty.AppInfoFactory; import com.google.common.flogger.GoogleLogger; import java.io.PrintWriter; import java.io.StringWriter; import java.time.Duration; import java.util.concurrent.TimeoutException; import org.eclipse.jetty.server.Handler; -import org.eclipse.jetty.server.HttpStream; import org.eclipse.jetty.server.Request; import org.eclipse.jetty.server.Response; -import org.eclipse.jetty.server.Server; import org.eclipse.jetty.util.Blocker; import org.eclipse.jetty.util.Callback; diff --git a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/http/JettyRequestAPIData.java b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/http/JettyRequestAPIData.java index 75d2e3a79..4b09f7275 100644 --- a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/http/JettyRequestAPIData.java +++ b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/http/JettyRequestAPIData.java @@ -55,9 +55,9 @@ import com.google.apphosting.base.protos.HttpPb; import com.google.apphosting.base.protos.RuntimePb; import com.google.apphosting.base.protos.TracePb; +import com.google.apphosting.runtime.AppInfoFactory; import com.google.apphosting.runtime.RequestAPIData; import com.google.apphosting.runtime.TraceContextHelper; -import com.google.apphosting.runtime.jetty.AppInfoFactory; import com.google.common.base.Strings; import com.google.common.flogger.GoogleLogger; import java.net.InetSocketAddress; diff --git a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/proxy/JettyHttpProxy.java b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/proxy/JettyHttpProxy.java index 5cfd8baba..08ee7d201 100644 --- a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/proxy/JettyHttpProxy.java +++ b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/proxy/JettyHttpProxy.java @@ -22,9 +22,9 @@ import com.google.apphosting.base.protos.RuntimePb.UPResponse; import com.google.apphosting.runtime.AppEngineConstants; import com.google.apphosting.runtime.LocalRpcContext; +import com.google.apphosting.runtime.AppInfoFactory; import com.google.apphosting.runtime.ServletEngineAdapter; import com.google.apphosting.runtime.anyrpc.EvaluationRuntimeServerInterface; -import com.google.apphosting.runtime.jetty.AppInfoFactory; import com.google.apphosting.runtime.jetty.AppVersionHandlerFactory; import com.google.common.base.Ascii; import com.google.common.base.Throwables; diff --git a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/proxy/UPRequestTranslator.java b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/proxy/UPRequestTranslator.java index 02c49757a..1e77cddbd 100644 --- a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/proxy/UPRequestTranslator.java +++ b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/proxy/UPRequestTranslator.java @@ -56,8 +56,8 @@ import com.google.apphosting.base.protos.RuntimePb; import com.google.apphosting.base.protos.RuntimePb.UPRequest; import com.google.apphosting.base.protos.TracePb.TraceContextProto; +import com.google.apphosting.runtime.AppInfoFactory; import com.google.apphosting.runtime.TraceContextHelper; -import com.google.apphosting.runtime.jetty.AppInfoFactory; import com.google.common.base.Ascii; import com.google.common.base.Strings; import com.google.common.flogger.GoogleLogger; diff --git a/runtime/runtime_impl_jetty12/src/test/java/com/google/apphosting/runtime/jetty/AppInfoFactoryTest.java b/runtime/runtime_impl_jetty12/src/test/java/com/google/apphosting/runtime/jetty/AppInfoFactoryTest.java deleted file mode 100644 index 0afb0f0a5..000000000 --- a/runtime/runtime_impl_jetty12/src/test/java/com/google/apphosting/runtime/jetty/AppInfoFactoryTest.java +++ /dev/null @@ -1,243 +0,0 @@ -/* - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.apphosting.runtime.jetty; - -import static com.google.common.base.StandardSystemProperty.USER_DIR; -import static com.google.common.truth.Truth.assertThat; -import static java.nio.charset.StandardCharsets.UTF_8; -import static org.junit.Assert.assertThrows; - -import com.google.appengine.tools.development.resource.ResourceExtractor; -import com.google.apphosting.base.protos.AppinfoPb; -import com.google.apphosting.utils.config.AppYaml; -import com.google.common.collect.ImmutableMap; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStreamReader; -import java.nio.file.NoSuchFileException; -import java.nio.file.Path; -import java.nio.file.Paths; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -@RunWith(JUnit4.class) -public final class AppInfoFactoryTest { - private static final String PACKAGE_PATH = - AppInfoFactoryTest.class.getPackage().getName().replace('.', '/'); - private static final String PROJECT_RESOURCE_NAME = - String.format("%s/mytestproject", PACKAGE_PATH); - - @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); - - private String appRoot; - private String fixedAppDir; - - @Before - public void setUp() throws IOException { - Path projPath = Paths.get(temporaryFolder.newFolder(PROJECT_RESOURCE_NAME).getPath()); - appRoot = projPath.getParent().toString(); - fixedAppDir = Paths.get(projPath.toString(), "100.mydeployment").toString(); - ResourceExtractor.toFile(PROJECT_RESOURCE_NAME, projPath.toString()); - } - - @Test - public void getGaeService_nonDefault() throws Exception { - AppInfoFactory factory = - new AppInfoFactory(ImmutableMap.of("GAE_SERVICE", "mytestservice")); - assertThat(factory.getGaeService()).isEqualTo("mytestservice"); - } - - @Test - public void getGaeService_defaults() throws Exception { - AppInfoFactory factory = new AppInfoFactory(ImmutableMap.of()); - assertThat(factory.getGaeService()).isEqualTo("default"); - } - - @Test - public void getGaeVersion_nonDefaultWithDeploymentId() throws Exception { - AppInfoFactory factory = - new AppInfoFactory( - ImmutableMap.of( - "GAE_SERVICE", "mytestservice", - "GAE_DEPLOYMENT_ID", "mydeployment", - "GAE_VERSION", "100")); - assertThat(factory.getGaeVersion()).isEqualTo("mytestservice:100.mydeployment"); - } - - @Test - public void getGaeVersion_defaultWithDeploymentId() throws Exception { - AppInfoFactory factory = - new AppInfoFactory( - ImmutableMap.of( - "GAE_DEPLOYMENT_ID", "mydeployment", - "GAE_VERSION", "100")); - assertThat(factory.getGaeVersion()).isEqualTo("100.mydeployment"); - } - - @Test - public void getGaeVersion_defaultWithoutDeploymentId() throws Exception { - AppInfoFactory factory = new AppInfoFactory(ImmutableMap.of("GAE_VERSION", "100")); - assertThat(factory.getGaeVersion()).isEqualTo("100"); - } - - @Test - public void getGaeServiceVersion_withDeploymentId() throws Exception { - AppInfoFactory factory = - new AppInfoFactory( - ImmutableMap.of( - "GAE_DEPLOYMENT_ID", "mydeployment", - "GAE_VERSION", "100")); - assertThat(factory.getGaeVersion()).isEqualTo("100.mydeployment"); - } - - @Test - public void getGaeServiceVersion_withoutDeploymentId() throws Exception { - AppInfoFactory factory = new AppInfoFactory(ImmutableMap.of("GAE_VERSION", "100")); - assertThat(factory.getGaeVersion()).isEqualTo("100"); - } - - @Test - public void getGaeApplication_nonDefault() throws Exception { - AppInfoFactory factory = new AppInfoFactory(ImmutableMap.of("GAE_APPLICATION", "s~myapp")); - assertThat(factory.getGaeApplication()).isEqualTo("s~myapp"); - } - - @Test - public void getGaeApplication_defaults() throws Exception { - AppInfoFactory factory = new AppInfoFactory(ImmutableMap.of()); - assertThat(factory.getGaeApplication()).isEqualTo("s~testapp"); - } - - @Test - public void getAppInfo_fixedApplicationPath() throws Exception { - AppInfoFactory factory = - new AppInfoFactory( - ImmutableMap.of( - "GAE_SERVICE", "mytestservice", - "GAE_DEPLOYMENT_ID", "mydeployment", - "GAE_VERSION", "100", - "GAE_APPLICATION", "s~myapp")); - AppinfoPb.AppInfo appInfo = factory.getAppInfoFromFile(null, fixedAppDir); - - assertThat(appInfo.getAppId()).isEqualTo("s~myapp"); - assertThat(appInfo.getVersionId()).isEqualTo("mytestservice:100.mydeployment"); - assertThat(appInfo.getRuntimeId()).isEqualTo("java8"); - assertThat(appInfo.getApiVersion()).isEqualTo("200"); - } - - @Test - public void getAppInfo_appRoot() throws Exception { - AppInfoFactory factory = - new AppInfoFactory( - ImmutableMap.of( - "GAE_SERVICE", "mytestservice", - "GAE_DEPLOYMENT_ID", "mydeployment", - "GAE_VERSION", "100", - "GAE_APPLICATION", "s~myapp", - "GOOGLE_CLOUD_PROJECT", "mytestproject")); - AppinfoPb.AppInfo appInfo = factory.getAppInfoFromFile(appRoot, null); - - assertThat(appInfo.getAppId()).isEqualTo("s~myapp"); - assertThat(appInfo.getVersionId()).isEqualTo("mytestservice:100.mydeployment"); - assertThat(appInfo.getRuntimeId()).isEqualTo("java8"); - assertThat(appInfo.getApiVersion()).isEqualTo("200"); - } - - @Test - public void getAppInfo_noAppYaml() throws Exception { - AppInfoFactory factory = - new AppInfoFactory( - ImmutableMap.of( - "GAE_SERVICE", "mytestservice", - "GAE_DEPLOYMENT_ID", "mydeployment", - "GAE_VERSION", "100", - "GAE_APPLICATION", "s~myapp", - "GOOGLE_CLOUD_PROJECT", "bogusproject")); - AppinfoPb.AppInfo appInfo = - factory.getAppInfoFromFile( - null, - // We tell AppInfoFactory to look directly in the current working directory. There's no - // app.yaml there: - USER_DIR.value()); - - assertThat(appInfo.getAppId()).isEqualTo("s~myapp"); - assertThat(appInfo.getVersionId()).isEqualTo("mytestservice:100.mydeployment"); - assertThat(appInfo.getRuntimeId()).isEqualTo("java8"); - assertThat(appInfo.getApiVersion()).isEmpty(); - } - - @Test - public void getAppInfo_noDirectory() throws Exception { - AppInfoFactory factory = - new AppInfoFactory( - ImmutableMap.of( - "GAE_SERVICE", "mytestservice", - "GAE_DEPLOYMENT_ID", "mydeployment", - "GAE_VERSION", "100", - "GAE_APPLICATION", "s~myapp", - // This will make the AppInfoFactory hunt for a directory called bogusproject: - "GOOGLE_CLOUD_PROJECT", "bogusproject")); - - assertThrows(NoSuchFileException.class, () -> factory.getAppInfoFromFile(appRoot, null)); - } - - @Test - public void getAppInfo_givenAppYaml() throws Exception { - AppInfoFactory factory = - new AppInfoFactory( - ImmutableMap.of( - "GAE_SERVICE", "mytestservice", - "GAE_DEPLOYMENT_ID", "mydeployment", - "GAE_VERSION", "100", - "GAE_APPLICATION", "s~myapp", - "GOOGLE_CLOUD_PROJECT", "mytestproject")); - - File appYamlFile = new File(fixedAppDir + "/WEB-INF/appengine-generated/app.yaml"); - AppYaml appYaml = AppYaml.parse(new InputStreamReader(new FileInputStream(appYamlFile), UTF_8)); - - AppinfoPb.AppInfo appInfo = factory.getAppInfoFromAppYaml(appYaml); - - assertThat(appInfo.getAppId()).isEqualTo("s~myapp"); - assertThat(appInfo.getVersionId()).isEqualTo("mytestservice:100.mydeployment"); - assertThat(appInfo.getRuntimeId()).isEqualTo("java8"); - assertThat(appInfo.getApiVersion()).isEqualTo("200"); - } - - @Test - public void getAppInfo_givenVersion() throws Exception { - AppInfoFactory factory = - new AppInfoFactory( - ImmutableMap.of( - "GAE_SERVICE", "mytestservice", - "GAE_DEPLOYMENT_ID", "mydeployment", - "GAE_VERSION", "100", - "GAE_APPLICATION", "s~myapp", - "GOOGLE_CLOUD_PROJECT", "mytestproject")); - - AppinfoPb.AppInfo appInfo = factory.getAppInfoWithApiVersion("my_api_version"); - - assertThat(appInfo.getAppId()).isEqualTo("s~myapp"); - assertThat(appInfo.getVersionId()).isEqualTo("mytestservice:100.mydeployment"); - assertThat(appInfo.getRuntimeId()).isEqualTo("java8"); - assertThat(appInfo.getApiVersion()).isEqualTo("my_api_version"); - } -} diff --git a/runtime/runtime_impl_jetty12/src/test/java/com/google/apphosting/runtime/jetty/UPRequestTranslatorTest.java b/runtime/runtime_impl_jetty12/src/test/java/com/google/apphosting/runtime/jetty/UPRequestTranslatorTest.java index 1755b6a48..8d01e177b 100644 --- a/runtime/runtime_impl_jetty12/src/test/java/com/google/apphosting/runtime/jetty/UPRequestTranslatorTest.java +++ b/runtime/runtime_impl_jetty12/src/test/java/com/google/apphosting/runtime/jetty/UPRequestTranslatorTest.java @@ -32,6 +32,7 @@ import com.google.apphosting.base.protos.RuntimePb; import com.google.apphosting.base.protos.TraceId.TraceIdProto; import com.google.apphosting.base.protos.TracePb.TraceContextProto; +import com.google.apphosting.runtime.AppInfoFactory; import com.google.apphosting.runtime.jetty.proxy.UPRequestTranslator; import com.google.common.base.Ascii; import com.google.common.collect.ImmutableMap; diff --git a/runtime/runtime_impl_jetty121/src/main/java/com/google/apphosting/runtime/jetty/AppInfoFactory.java b/runtime/runtime_impl_jetty121/src/main/java/com/google/apphosting/runtime/jetty/AppInfoFactory.java deleted file mode 100644 index 7ceb0ef3e..000000000 --- a/runtime/runtime_impl_jetty121/src/main/java/com/google/apphosting/runtime/jetty/AppInfoFactory.java +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.apphosting.runtime.jetty; - -import static java.nio.charset.StandardCharsets.UTF_8; - -import com.esotericsoftware.yamlbeans.YamlReader; -import com.google.apphosting.base.protos.AppinfoPb; -import com.google.apphosting.utils.config.AppYaml; -import com.google.common.flogger.GoogleLogger; -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.NoSuchFileException; -import java.util.Map; -import org.jspecify.annotations.Nullable; - -/** Builds AppinfoPb.AppInfo from the given ServletEngineAdapter.Config and environment. */ -public class AppInfoFactory { - - private static final GoogleLogger logger = GoogleLogger.forEnclosingClass(); - - private static final String DEFAULT_CLOUD_PROJECT = "testapp"; - private static final String DEFAULT_GAE_APPLICATION = "s~testapp"; - private static final String DEFAULT_GAE_SERVICE = "default"; - private static final String DEFAULT_GAE_VERSION = "1.0"; - - /** Path in the WAR layout to app.yaml */ - private static final String APP_YAML_PATH = "WEB-INF/appengine-generated/app.yaml"; - - private final String gaeVersion; - private final String googleCloudProject; - private final String gaeApplication; - private final String gaeService; - private final String gaeServiceVersion; - - public AppInfoFactory(Map env) { - String version = env.getOrDefault("GAE_VERSION", DEFAULT_GAE_VERSION); - String deploymentId = env.getOrDefault("GAE_DEPLOYMENT_ID", null); - gaeServiceVersion = (deploymentId != null) ? version + "." + deploymentId : version; - gaeService = env.getOrDefault("GAE_SERVICE", DEFAULT_GAE_SERVICE); - // Prepend service if it exists, otherwise do not prepend DEFAULT (go/app-engine-ids) - gaeVersion = - DEFAULT_GAE_SERVICE.equals(this.gaeService) - ? this.gaeServiceVersion - : this.gaeService + ":" + this.gaeServiceVersion; - googleCloudProject = env.getOrDefault("GOOGLE_CLOUD_PROJECT", DEFAULT_CLOUD_PROJECT); - gaeApplication = env.getOrDefault("GAE_APPLICATION", DEFAULT_GAE_APPLICATION); - } - - public String getGaeService() { - return gaeService; - } - - public String getGaeVersion() { - return gaeVersion; - } - - public String getGaeServiceVersion() { - return gaeServiceVersion; - } - - public String getGaeApplication() { - return gaeApplication; - } - - /** Creates a AppinfoPb.AppInfo object. */ - public AppinfoPb.AppInfo getAppInfoFromFile(String applicationRoot, String fixedApplicationPath) - throws IOException { - // App should be located under /base/data/home/apps/appId/versionID or in the optional - // fixedApplicationPath parameter. - String applicationPath = - (fixedApplicationPath == null) - ? applicationRoot + "/" + googleCloudProject + "/" + gaeServiceVersion - : fixedApplicationPath; - - if (!new File(applicationPath).exists()) { - throw new NoSuchFileException("Application does not exist under: " + applicationPath); - } - @Nullable String apiVersion = null; - File appYamlFile = new File(applicationPath, APP_YAML_PATH); - try { - YamlReader reader = new YamlReader(Files.newBufferedReader(appYamlFile.toPath(), UTF_8)); - Object apiVersionObj = ((Map) reader.read()).get("api_version"); - if (apiVersionObj != null) { - apiVersion = (String) apiVersionObj; - } - } catch (NoSuchFileException ex) { - logger.atInfo().log( - "Cannot configure App Engine APIs, because the generated app.yaml file " - + "does not exist: %s", - appYamlFile.getAbsolutePath()); - } - return getAppInfoWithApiVersion(apiVersion); - } - - public AppinfoPb.AppInfo getAppInfoFromAppYaml(AppYaml appYaml) throws IOException { - return getAppInfoWithApiVersion(appYaml.getApi_version()); - } - - public AppinfoPb.AppInfo getAppInfoWithApiVersion(@Nullable String apiVersion) { - final AppinfoPb.AppInfo.Builder appInfoBuilder = - AppinfoPb.AppInfo.newBuilder() - .setAppId(gaeApplication) - .setVersionId(gaeVersion) - .setRuntimeId("java8"); - - if (apiVersion != null) { - appInfoBuilder.setApiVersion(apiVersion); - } - - return appInfoBuilder.build(); - } -} diff --git a/runtime/runtime_impl_jetty121/src/main/java/com/google/apphosting/runtime/jetty/JettyServletEngineAdapter.java b/runtime/runtime_impl_jetty121/src/main/java/com/google/apphosting/runtime/jetty/JettyServletEngineAdapter.java index bc14eaaa9..9bdefcfa8 100644 --- a/runtime/runtime_impl_jetty121/src/main/java/com/google/apphosting/runtime/jetty/JettyServletEngineAdapter.java +++ b/runtime/runtime_impl_jetty121/src/main/java/com/google/apphosting/runtime/jetty/JettyServletEngineAdapter.java @@ -28,6 +28,7 @@ import com.google.apphosting.base.protos.RuntimePb.UPRequest; import com.google.apphosting.base.protos.RuntimePb.UPResponse; import com.google.apphosting.runtime.AppEngineConstants; +import com.google.apphosting.runtime.AppInfoFactory; import com.google.apphosting.runtime.AppVersion; import com.google.apphosting.runtime.LocalRpcContext; import com.google.apphosting.runtime.MutableUpResponse; diff --git a/runtime/runtime_impl_jetty121/src/main/java/com/google/apphosting/runtime/jetty/http/JettyHttpHandler.java b/runtime/runtime_impl_jetty121/src/main/java/com/google/apphosting/runtime/jetty/http/JettyHttpHandler.java index 9498b745e..7dcba3b70 100644 --- a/runtime/runtime_impl_jetty121/src/main/java/com/google/apphosting/runtime/jetty/http/JettyHttpHandler.java +++ b/runtime/runtime_impl_jetty121/src/main/java/com/google/apphosting/runtime/jetty/http/JettyHttpHandler.java @@ -32,9 +32,9 @@ import com.google.apphosting.runtime.RequestRunner; import com.google.apphosting.runtime.RequestRunner.EagerRunner; import com.google.apphosting.runtime.ResponseAPIData; +import com.google.apphosting.runtime.AppInfoFactory; import com.google.apphosting.runtime.ServletEngineAdapter; import com.google.apphosting.runtime.anyrpc.AnyRpcServerContext; -import com.google.apphosting.runtime.jetty.AppInfoFactory; import com.google.common.flogger.GoogleLogger; import java.io.PrintWriter; import java.io.StringWriter; @@ -170,7 +170,7 @@ private boolean dispatchRequest( requestManager.shutdownRequests(requestToken); return true; case BACKGROUND: - dispatchBackgroundRequest(request, response); + dispatchBackgroundRequest(request); return true; case OTHER: return dispatchServletRequest(request, response); @@ -194,7 +194,7 @@ private boolean dispatchServletRequest(JettyRequestAPIData request, JettyRespons } } - private void dispatchBackgroundRequest(JettyRequestAPIData request, JettyResponseAPIData response) + private void dispatchBackgroundRequest(JettyRequestAPIData request) throws InterruptedException, TimeoutException { String requestId = getBackgroundRequestId(request); // The interface of coordinator.waitForUserRunnable() requires us to provide the app code with a diff --git a/runtime/runtime_impl_jetty121/src/main/java/com/google/apphosting/runtime/jetty/http/JettyRequestAPIData.java b/runtime/runtime_impl_jetty121/src/main/java/com/google/apphosting/runtime/jetty/http/JettyRequestAPIData.java index 75d2e3a79..4b09f7275 100644 --- a/runtime/runtime_impl_jetty121/src/main/java/com/google/apphosting/runtime/jetty/http/JettyRequestAPIData.java +++ b/runtime/runtime_impl_jetty121/src/main/java/com/google/apphosting/runtime/jetty/http/JettyRequestAPIData.java @@ -55,9 +55,9 @@ import com.google.apphosting.base.protos.HttpPb; import com.google.apphosting.base.protos.RuntimePb; import com.google.apphosting.base.protos.TracePb; +import com.google.apphosting.runtime.AppInfoFactory; import com.google.apphosting.runtime.RequestAPIData; import com.google.apphosting.runtime.TraceContextHelper; -import com.google.apphosting.runtime.jetty.AppInfoFactory; import com.google.common.base.Strings; import com.google.common.flogger.GoogleLogger; import java.net.InetSocketAddress; diff --git a/runtime/runtime_impl_jetty121/src/main/java/com/google/apphosting/runtime/jetty/proxy/JettyHttpProxy.java b/runtime/runtime_impl_jetty121/src/main/java/com/google/apphosting/runtime/jetty/proxy/JettyHttpProxy.java index e52a75525..5d35902ca 100644 --- a/runtime/runtime_impl_jetty121/src/main/java/com/google/apphosting/runtime/jetty/proxy/JettyHttpProxy.java +++ b/runtime/runtime_impl_jetty121/src/main/java/com/google/apphosting/runtime/jetty/proxy/JettyHttpProxy.java @@ -22,9 +22,9 @@ import com.google.apphosting.base.protos.RuntimePb.UPResponse; import com.google.apphosting.runtime.AppEngineConstants; import com.google.apphosting.runtime.LocalRpcContext; +import com.google.apphosting.runtime.AppInfoFactory; import com.google.apphosting.runtime.ServletEngineAdapter; import com.google.apphosting.runtime.anyrpc.EvaluationRuntimeServerInterface; -import com.google.apphosting.runtime.jetty.AppInfoFactory; import com.google.apphosting.runtime.jetty.AppVersionHandlerFactory; import com.google.common.base.Ascii; import com.google.common.base.Throwables; diff --git a/runtime/runtime_impl_jetty121/src/main/java/com/google/apphosting/runtime/jetty/proxy/UPRequestTranslator.java b/runtime/runtime_impl_jetty121/src/main/java/com/google/apphosting/runtime/jetty/proxy/UPRequestTranslator.java index 02c49757a..1e77cddbd 100644 --- a/runtime/runtime_impl_jetty121/src/main/java/com/google/apphosting/runtime/jetty/proxy/UPRequestTranslator.java +++ b/runtime/runtime_impl_jetty121/src/main/java/com/google/apphosting/runtime/jetty/proxy/UPRequestTranslator.java @@ -56,8 +56,8 @@ import com.google.apphosting.base.protos.RuntimePb; import com.google.apphosting.base.protos.RuntimePb.UPRequest; import com.google.apphosting.base.protos.TracePb.TraceContextProto; +import com.google.apphosting.runtime.AppInfoFactory; import com.google.apphosting.runtime.TraceContextHelper; -import com.google.apphosting.runtime.jetty.AppInfoFactory; import com.google.common.base.Ascii; import com.google.common.base.Strings; import com.google.common.flogger.GoogleLogger; diff --git a/runtime/runtime_impl_jetty121/src/test/java/com/google/apphosting/runtime/jetty/AppInfoFactoryTest.java b/runtime/runtime_impl_jetty121/src/test/java/com/google/apphosting/runtime/jetty/AppInfoFactoryTest.java deleted file mode 100644 index 0f142f7d6..000000000 --- a/runtime/runtime_impl_jetty121/src/test/java/com/google/apphosting/runtime/jetty/AppInfoFactoryTest.java +++ /dev/null @@ -1,242 +0,0 @@ -/* - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.apphosting.runtime.jetty; - -import static com.google.common.base.StandardSystemProperty.USER_DIR; -import static com.google.common.truth.Truth.assertThat; -import static java.nio.charset.StandardCharsets.UTF_8; -import static org.junit.Assert.assertThrows; - -import com.google.appengine.tools.development.resource.ResourceExtractor; -import com.google.apphosting.base.protos.AppinfoPb; -import com.google.apphosting.utils.config.AppYaml; -import com.google.common.collect.ImmutableMap; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStreamReader; -import java.nio.file.NoSuchFileException; -import java.nio.file.Path; -import java.nio.file.Paths; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -@RunWith(JUnit4.class) -public final class AppInfoFactoryTest { - private static final String PACKAGE_PATH = - AppInfoFactoryTest.class.getPackage().getName().replace('.', '/'); - private static final String PROJECT_RESOURCE_NAME = - String.format("%s/mytestproject", PACKAGE_PATH); - - @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); - - private String appRoot; - private String fixedAppDir; - - @Before - public void setUp() throws IOException { - Path projPath = Paths.get(temporaryFolder.newFolder(PROJECT_RESOURCE_NAME).getPath()); - appRoot = projPath.getParent().toString(); - fixedAppDir = Paths.get(projPath.toString(), "100.mydeployment").toString(); - ResourceExtractor.toFile(PROJECT_RESOURCE_NAME, projPath.toString()); - } - - @Test - public void getGaeService_nonDefault() throws Exception { - AppInfoFactory factory = new AppInfoFactory(ImmutableMap.of("GAE_SERVICE", "mytestservice")); - assertThat(factory.getGaeService()).isEqualTo("mytestservice"); - } - - @Test - public void getGaeService_defaults() throws Exception { - AppInfoFactory factory = new AppInfoFactory(ImmutableMap.of()); - assertThat(factory.getGaeService()).isEqualTo("default"); - } - - @Test - public void getGaeVersion_nonDefaultWithDeploymentId() throws Exception { - AppInfoFactory factory = - new AppInfoFactory( - ImmutableMap.of( - "GAE_SERVICE", "mytestservice", - "GAE_DEPLOYMENT_ID", "mydeployment", - "GAE_VERSION", "100")); - assertThat(factory.getGaeVersion()).isEqualTo("mytestservice:100.mydeployment"); - } - - @Test - public void getGaeVersion_defaultWithDeploymentId() throws Exception { - AppInfoFactory factory = - new AppInfoFactory( - ImmutableMap.of( - "GAE_DEPLOYMENT_ID", "mydeployment", - "GAE_VERSION", "100")); - assertThat(factory.getGaeVersion()).isEqualTo("100.mydeployment"); - } - - @Test - public void getGaeVersion_defaultWithoutDeploymentId() throws Exception { - AppInfoFactory factory = new AppInfoFactory(ImmutableMap.of("GAE_VERSION", "100")); - assertThat(factory.getGaeVersion()).isEqualTo("100"); - } - - @Test - public void getGaeServiceVersion_withDeploymentId() throws Exception { - AppInfoFactory factory = - new AppInfoFactory( - ImmutableMap.of( - "GAE_DEPLOYMENT_ID", "mydeployment", - "GAE_VERSION", "100")); - assertThat(factory.getGaeVersion()).isEqualTo("100.mydeployment"); - } - - @Test - public void getGaeServiceVersion_withoutDeploymentId() throws Exception { - AppInfoFactory factory = new AppInfoFactory(ImmutableMap.of("GAE_VERSION", "100")); - assertThat(factory.getGaeVersion()).isEqualTo("100"); - } - - @Test - public void getGaeApplication_nonDefault() throws Exception { - AppInfoFactory factory = new AppInfoFactory(ImmutableMap.of("GAE_APPLICATION", "s~myapp")); - assertThat(factory.getGaeApplication()).isEqualTo("s~myapp"); - } - - @Test - public void getGaeApplication_defaults() throws Exception { - AppInfoFactory factory = new AppInfoFactory(ImmutableMap.of()); - assertThat(factory.getGaeApplication()).isEqualTo("s~testapp"); - } - - @Test - public void getAppInfo_fixedApplicationPath() throws Exception { - AppInfoFactory factory = - new AppInfoFactory( - ImmutableMap.of( - "GAE_SERVICE", "mytestservice", - "GAE_DEPLOYMENT_ID", "mydeployment", - "GAE_VERSION", "100", - "GAE_APPLICATION", "s~myapp")); - AppinfoPb.AppInfo appInfo = factory.getAppInfoFromFile(null, fixedAppDir); - - assertThat(appInfo.getAppId()).isEqualTo("s~myapp"); - assertThat(appInfo.getVersionId()).isEqualTo("mytestservice:100.mydeployment"); - assertThat(appInfo.getRuntimeId()).isEqualTo("java8"); - assertThat(appInfo.getApiVersion()).isEqualTo("200"); - } - - @Test - public void getAppInfo_appRoot() throws Exception { - AppInfoFactory factory = - new AppInfoFactory( - ImmutableMap.of( - "GAE_SERVICE", "mytestservice", - "GAE_DEPLOYMENT_ID", "mydeployment", - "GAE_VERSION", "100", - "GAE_APPLICATION", "s~myapp", - "GOOGLE_CLOUD_PROJECT", "mytestproject")); - AppinfoPb.AppInfo appInfo = factory.getAppInfoFromFile(appRoot, null); - - assertThat(appInfo.getAppId()).isEqualTo("s~myapp"); - assertThat(appInfo.getVersionId()).isEqualTo("mytestservice:100.mydeployment"); - assertThat(appInfo.getRuntimeId()).isEqualTo("java8"); - assertThat(appInfo.getApiVersion()).isEqualTo("200"); - } - - @Test - public void getAppInfo_noAppYaml() throws Exception { - AppInfoFactory factory = - new AppInfoFactory( - ImmutableMap.of( - "GAE_SERVICE", "mytestservice", - "GAE_DEPLOYMENT_ID", "mydeployment", - "GAE_VERSION", "100", - "GAE_APPLICATION", "s~myapp", - "GOOGLE_CLOUD_PROJECT", "bogusproject")); - AppinfoPb.AppInfo appInfo = - factory.getAppInfoFromFile( - null, - // We tell AppInfoFactory to look directly in the current working directory. There's no - // app.yaml there: - USER_DIR.value()); - - assertThat(appInfo.getAppId()).isEqualTo("s~myapp"); - assertThat(appInfo.getVersionId()).isEqualTo("mytestservice:100.mydeployment"); - assertThat(appInfo.getRuntimeId()).isEqualTo("java8"); - assertThat(appInfo.getApiVersion()).isEmpty(); - } - - @Test - public void getAppInfo_noDirectory() throws Exception { - AppInfoFactory factory = - new AppInfoFactory( - ImmutableMap.of( - "GAE_SERVICE", "mytestservice", - "GAE_DEPLOYMENT_ID", "mydeployment", - "GAE_VERSION", "100", - "GAE_APPLICATION", "s~myapp", - // This will make the AppInfoFactory hunt for a directory called bogusproject: - "GOOGLE_CLOUD_PROJECT", "bogusproject")); - - assertThrows(NoSuchFileException.class, () -> factory.getAppInfoFromFile(appRoot, null)); - } - - @Test - public void getAppInfo_givenAppYaml() throws Exception { - AppInfoFactory factory = - new AppInfoFactory( - ImmutableMap.of( - "GAE_SERVICE", "mytestservice", - "GAE_DEPLOYMENT_ID", "mydeployment", - "GAE_VERSION", "100", - "GAE_APPLICATION", "s~myapp", - "GOOGLE_CLOUD_PROJECT", "mytestproject")); - - File appYamlFile = new File(fixedAppDir + "/WEB-INF/appengine-generated/app.yaml"); - AppYaml appYaml = AppYaml.parse(new InputStreamReader(new FileInputStream(appYamlFile), UTF_8)); - - AppinfoPb.AppInfo appInfo = factory.getAppInfoFromAppYaml(appYaml); - - assertThat(appInfo.getAppId()).isEqualTo("s~myapp"); - assertThat(appInfo.getVersionId()).isEqualTo("mytestservice:100.mydeployment"); - assertThat(appInfo.getRuntimeId()).isEqualTo("java8"); - assertThat(appInfo.getApiVersion()).isEqualTo("200"); - } - - @Test - public void getAppInfo_givenVersion() throws Exception { - AppInfoFactory factory = - new AppInfoFactory( - ImmutableMap.of( - "GAE_SERVICE", "mytestservice", - "GAE_DEPLOYMENT_ID", "mydeployment", - "GAE_VERSION", "100", - "GAE_APPLICATION", "s~myapp", - "GOOGLE_CLOUD_PROJECT", "mytestproject")); - - AppinfoPb.AppInfo appInfo = factory.getAppInfoWithApiVersion("my_api_version"); - - assertThat(appInfo.getAppId()).isEqualTo("s~myapp"); - assertThat(appInfo.getVersionId()).isEqualTo("mytestservice:100.mydeployment"); - assertThat(appInfo.getRuntimeId()).isEqualTo("java8"); - assertThat(appInfo.getApiVersion()).isEqualTo("my_api_version"); - } -} diff --git a/runtime/runtime_impl_jetty121/src/test/java/com/google/apphosting/runtime/jetty/UPRequestTranslatorTest.java b/runtime/runtime_impl_jetty121/src/test/java/com/google/apphosting/runtime/jetty/UPRequestTranslatorTest.java index 85df696fa..4abefab54 100644 --- a/runtime/runtime_impl_jetty121/src/test/java/com/google/apphosting/runtime/jetty/UPRequestTranslatorTest.java +++ b/runtime/runtime_impl_jetty121/src/test/java/com/google/apphosting/runtime/jetty/UPRequestTranslatorTest.java @@ -32,24 +32,19 @@ import com.google.apphosting.base.protos.RuntimePb; import com.google.apphosting.base.protos.TraceId.TraceIdProto; import com.google.apphosting.base.protos.TracePb.TraceContextProto; +import com.google.apphosting.runtime.AppInfoFactory; import com.google.apphosting.runtime.jetty.proxy.UPRequestTranslator; import com.google.common.base.Ascii; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.protobuf.ExtensionRegistry; import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.OutputStream; import java.net.SocketAddress; import java.net.URI; import java.net.URISyntaxException; import java.nio.ByteBuffer; import java.util.HashMap; import java.util.Map; -import javax.servlet.ReadListener; -import javax.servlet.ServletInputStream; -import javax.servlet.ServletOutputStream; -import javax.servlet.WriteListener; import javax.servlet.http.HttpServletResponse; import org.eclipse.jetty.http.HttpFields; import org.eclipse.jetty.http.HttpURI; @@ -68,6 +63,7 @@ import org.mockito.stubbing.Answer; @RunWith(JUnit4.class) +@SuppressWarnings("GoogleHttpHeaderConstants") public final class UPRequestTranslatorTest { private static final String X_APPENGINE_HTTPS = "X-AppEngine-Https"; private static final String X_APPENGINE_USER_IP = "X-AppEngine-User-IP"; @@ -463,46 +459,4 @@ private static Request mockServletRequest( return httpRequest; } - private static ServletInputStream emptyInputStream() { - return new ServletInputStream() { - @Override - public int read() { - return -1; - } - - @Override - public void setReadListener(ReadListener listener) { - throw new UnsupportedOperationException(); - } - - @Override - public boolean isReady() { - return true; - } - - @Override - public boolean isFinished() { - return true; - } - }; - } - - private static ServletOutputStream copyingOutputStream(OutputStream out) { - return new ServletOutputStream() { - @Override - public void write(int b) throws IOException { - out.write(b); - } - - @Override - public void setWriteListener(WriteListener listener) { - throw new UnsupportedOperationException(); - } - - @Override - public boolean isReady() { - return true; - } - }; - } } diff --git a/runtime/runtime_impl_jetty9/src/main/java/com/google/apphosting/runtime/jetty9/JettyHttpHandler.java b/runtime/runtime_impl_jetty9/src/main/java/com/google/apphosting/runtime/jetty9/JettyHttpHandler.java index e8f682364..3a6ba66be 100644 --- a/runtime/runtime_impl_jetty9/src/main/java/com/google/apphosting/runtime/jetty9/JettyHttpHandler.java +++ b/runtime/runtime_impl_jetty9/src/main/java/com/google/apphosting/runtime/jetty9/JettyHttpHandler.java @@ -26,11 +26,11 @@ import com.google.apphosting.base.protos.RuntimePb; import com.google.apphosting.runtime.ApiProxyImpl; import com.google.apphosting.runtime.AppEngineConstants; +import com.google.apphosting.runtime.AppInfoFactory; import com.google.apphosting.runtime.AppVersion; import com.google.apphosting.runtime.BackgroundRequestCoordinator; import com.google.apphosting.runtime.LocalRpcContext; import com.google.apphosting.runtime.RequestManager; -import com.google.apphosting.runtime.RequestRunner; import com.google.apphosting.runtime.RequestRunner.EagerRunner; import com.google.apphosting.runtime.ResponseAPIData; import com.google.apphosting.runtime.ServletEngineAdapter; @@ -41,19 +41,17 @@ import java.io.StringWriter; import java.time.Duration; import java.util.concurrent.TimeoutException; -import org.jspecify.annotations.Nullable; import javax.servlet.ServletException; import javax.servlet.UnavailableException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.eclipse.jetty.http.BadMessageException; import org.eclipse.jetty.http.HttpStatus; -import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.HttpChannel; import org.eclipse.jetty.server.Request; -import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.server.handler.HandlerWrapper; +import org.jspecify.annotations.Nullable; /** * This class replicates the behaviour of the {@link RequestRunner} for Requests which do not come diff --git a/runtime/runtime_impl_jetty9/src/main/java/com/google/apphosting/runtime/jetty9/JettyHttpProxy.java b/runtime/runtime_impl_jetty9/src/main/java/com/google/apphosting/runtime/jetty9/JettyHttpProxy.java index 9a2998a94..dbc7ef02e 100644 --- a/runtime/runtime_impl_jetty9/src/main/java/com/google/apphosting/runtime/jetty9/JettyHttpProxy.java +++ b/runtime/runtime_impl_jetty9/src/main/java/com/google/apphosting/runtime/jetty9/JettyHttpProxy.java @@ -24,6 +24,7 @@ import com.google.apphosting.base.protos.RuntimePb.UPRequest; import com.google.apphosting.base.protos.RuntimePb.UPResponse; import com.google.apphosting.runtime.LocalRpcContext; +import com.google.apphosting.runtime.AppInfoFactory; import com.google.apphosting.runtime.ServletEngineAdapter; import com.google.apphosting.runtime.anyrpc.EvaluationRuntimeServerInterface; import com.google.common.base.Ascii; diff --git a/runtime/runtime_impl_jetty9/src/main/java/com/google/apphosting/runtime/jetty9/JettyRequestAPIData.java b/runtime/runtime_impl_jetty9/src/main/java/com/google/apphosting/runtime/jetty9/JettyRequestAPIData.java index e4168f8c2..f12726bed 100644 --- a/runtime/runtime_impl_jetty9/src/main/java/com/google/apphosting/runtime/jetty9/JettyRequestAPIData.java +++ b/runtime/runtime_impl_jetty9/src/main/java/com/google/apphosting/runtime/jetty9/JettyRequestAPIData.java @@ -57,6 +57,7 @@ import com.google.apphosting.base.protos.RuntimePb; import com.google.apphosting.base.protos.TracePb; import com.google.apphosting.base.protos.TracePb.TraceContextProto; +import com.google.apphosting.runtime.AppInfoFactory; import com.google.apphosting.runtime.RequestAPIData; import com.google.apphosting.runtime.TraceContextHelper; import com.google.common.base.Strings; diff --git a/runtime/runtime_impl_jetty9/src/main/java/com/google/apphosting/runtime/jetty9/JettyServletEngineAdapter.java b/runtime/runtime_impl_jetty9/src/main/java/com/google/apphosting/runtime/jetty9/JettyServletEngineAdapter.java index 3e53a0a7b..ae76d47d8 100644 --- a/runtime/runtime_impl_jetty9/src/main/java/com/google/apphosting/runtime/jetty9/JettyServletEngineAdapter.java +++ b/runtime/runtime_impl_jetty9/src/main/java/com/google/apphosting/runtime/jetty9/JettyServletEngineAdapter.java @@ -27,6 +27,7 @@ import com.google.apphosting.base.protos.RuntimePb.UPRequest; import com.google.apphosting.base.protos.RuntimePb.UPResponse; import com.google.apphosting.runtime.AppVersion; +import com.google.apphosting.runtime.AppInfoFactory; import com.google.apphosting.runtime.LocalRpcContext; import com.google.apphosting.runtime.MutableUpResponse; import com.google.apphosting.runtime.ServletEngineAdapter; diff --git a/runtime/runtime_impl_jetty9/src/main/java/com/google/apphosting/runtime/jetty9/UPRequestTranslator.java b/runtime/runtime_impl_jetty9/src/main/java/com/google/apphosting/runtime/jetty9/UPRequestTranslator.java index 4673926df..62a0038cb 100644 --- a/runtime/runtime_impl_jetty9/src/main/java/com/google/apphosting/runtime/jetty9/UPRequestTranslator.java +++ b/runtime/runtime_impl_jetty9/src/main/java/com/google/apphosting/runtime/jetty9/UPRequestTranslator.java @@ -55,6 +55,7 @@ import com.google.apphosting.base.protos.RuntimePb; import com.google.apphosting.base.protos.RuntimePb.UPRequest; import com.google.apphosting.base.protos.TracePb.TraceContextProto; +import com.google.apphosting.runtime.AppInfoFactory; import com.google.apphosting.runtime.TraceContextHelper; import com.google.common.base.Ascii; import com.google.common.base.Strings; diff --git a/runtime/runtime_impl_jetty9/src/test/java/com/google/apphosting/runtime/jetty9/UPRequestTranslatorTest.java b/runtime/runtime_impl_jetty9/src/test/java/com/google/apphosting/runtime/jetty9/UPRequestTranslatorTest.java index 2d90b76b8..98d636988 100644 --- a/runtime/runtime_impl_jetty9/src/test/java/com/google/apphosting/runtime/jetty9/UPRequestTranslatorTest.java +++ b/runtime/runtime_impl_jetty9/src/test/java/com/google/apphosting/runtime/jetty9/UPRequestTranslatorTest.java @@ -32,6 +32,7 @@ import com.google.apphosting.base.protos.RuntimePb; import com.google.apphosting.base.protos.TraceId.TraceIdProto; import com.google.apphosting.base.protos.TracePb.TraceContextProto; +import com.google.apphosting.runtime.AppInfoFactory; import com.google.common.base.Ascii; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -58,6 +59,7 @@ import org.mockito.stubbing.Answer; @RunWith(JUnit4.class) +@SuppressWarnings("GoogleHttpHeaderConstants") public final class UPRequestTranslatorTest { private static final String X_APPENGINE_HTTPS = "X-AppEngine-Https"; private static final String X_APPENGINE_USER_IP = "X-AppEngine-User-IP";