diff --git a/api/src/main/java/com/google/appengine/api/blobstore/BlobstoreServiceImpl.java b/api/src/main/java/com/google/appengine/api/blobstore/BlobstoreServiceImpl.java index 059cf7d2a..8e9e86dd8 100644 --- a/api/src/main/java/com/google/appengine/api/blobstore/BlobstoreServiceImpl.java +++ b/api/src/main/java/com/google/appengine/api/blobstore/BlobstoreServiceImpl.java @@ -164,12 +164,10 @@ public void delete(BlobKey... blobKeys) { try { ApiProxy.makeSyncCall(PACKAGE, "DeleteBlob", request.build().toByteArray()); } catch (ApiProxy.ApplicationException ex) { - switch (BlobstoreServiceError.ErrorCode.forNumber(ex.getApplicationError())) { - case INTERNAL_ERROR: - throw new BlobstoreFailureException("An internal blobstore error occurred."); - default: - throw new BlobstoreFailureException("An unexpected error occurred.", ex); - } + throw switch (BlobstoreServiceError.ErrorCode.forNumber(ex.getApplicationError())) { + case INTERNAL_ERROR -> new BlobstoreFailureException("An internal blobstore error occurred."); + default -> new BlobstoreFailureException("An unexpected error occurred.", ex); + }; } } diff --git a/api/src/main/java/com/google/appengine/api/datastore/AsyncDatastoreServiceImpl.java b/api/src/main/java/com/google/appengine/api/datastore/AsyncDatastoreServiceImpl.java index 92ab170e3..eb8f11637 100644 --- a/api/src/main/java/com/google/appengine/api/datastore/AsyncDatastoreServiceImpl.java +++ b/api/src/main/java/com/google/appengine/api/datastore/AsyncDatastoreServiceImpl.java @@ -253,16 +253,11 @@ protected TransactionImpl.InternalTransaction doBeginTransaction(TransactionOpti } } if (options.transactionMode() != null) { - switch (options.transactionMode()) { - case READ_ONLY: - request.setMode(TransactionMode.READ_ONLY); - break; - case READ_WRITE: - request.setMode(TransactionMode.READ_WRITE); - break; - default: - throw new AssertionError("Unrecognized transaction mode: " + options.transactionMode()); - } + request.setMode(switch (options.transactionMode()) { + case READ_ONLY -> TransactionMode.READ_ONLY; + case READ_WRITE -> TransactionMode.READ_WRITE; + default -> throw new AssertionError("Unrecognized transaction mode: " + options.transactionMode()); + }); } Future future = @@ -620,21 +615,11 @@ protected Map wrap(CompositeIndices indices) throws Exception for (CompositeIndex ci : indices.getIndexList()) { Index index = IndexTranslator.convertFromPb(ci); switch (ci.getState()) { - case DELETED: - answer.put(index, IndexState.DELETING); - break; - case ERROR: - answer.put(index, IndexState.ERROR); - break; - case READ_WRITE: - answer.put(index, IndexState.SERVING); - break; - case WRITE_ONLY: - answer.put(index, IndexState.BUILDING); - break; - default: - logger.log(Level.WARNING, "Unrecognized index state for " + index); - break; + case DELETED -> answer.put(index, IndexState.DELETING); + case ERROR -> answer.put(index, IndexState.ERROR); + case READ_WRITE -> answer.put(index, IndexState.SERVING); + case WRITE_ONLY -> answer.put(index, IndexState.BUILDING); + default -> logger.log(Level.WARNING, "Unrecognized index state for {0}", index); } } return answer; diff --git a/api/src/main/java/com/google/appengine/api/datastore/BaseQueryResultsSource.java b/api/src/main/java/com/google/appengine/api/datastore/BaseQueryResultsSource.java index 0dea5f2fd..4d5d919ab 100644 --- a/api/src/main/java/com/google/appengine/api/datastore/BaseQueryResultsSource.java +++ b/api/src/main/java/com/google/appengine/api/datastore/BaseQueryResultsSource.java @@ -129,13 +129,7 @@ public BaseQueryResultsSource( this.offset = fetchOptions.getOffset() != null ? fetchOptions.getOffset() : 0; this.txn = txn; this.query = query; - this.currentTransactionProvider = - new CurrentTransactionProvider() { - @Override - public Transaction getCurrentTransaction(Transaction defaultValue) { - return txn; - } - }; + this.currentTransactionProvider = defaultValue -> txn; this.initialQueryResultFuture = initialQueryResultFuture; this.skippedResults = 0; } diff --git a/api/src/main/java/com/google/appengine/api/datastore/Blob.java b/api/src/main/java/com/google/appengine/api/datastore/Blob.java index f72b65df2..c7b51ca0d 100644 --- a/api/src/main/java/com/google/appengine/api/datastore/Blob.java +++ b/api/src/main/java/com/google/appengine/api/datastore/Blob.java @@ -62,8 +62,7 @@ public int hashCode() { /** Two {@code Blob} objects are considered equal if their contained bytes match exactly. */ @Override public boolean equals(@Nullable Object object) { - if (object instanceof Blob) { - Blob key = (Blob) object; + if (object instanceof Blob key) { return Arrays.equals(bytes, key.bytes); } return false; diff --git a/api/src/main/java/com/google/appengine/api/datastore/CloudDatastoreV1ClientImpl.java b/api/src/main/java/com/google/appengine/api/datastore/CloudDatastoreV1ClientImpl.java index 8f5d5d479..79ca0fd0c 100644 --- a/api/src/main/java/com/google/appengine/api/datastore/CloudDatastoreV1ClientImpl.java +++ b/api/src/main/java/com/google/appengine/api/datastore/CloudDatastoreV1ClientImpl.java @@ -22,8 +22,6 @@ import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; import com.google.api.client.googleapis.compute.ComputeCredential; import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; -import com.google.api.client.http.HttpRequest; -import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.json.gson.GsonFactory; import com.google.api.client.util.ExponentialBackOff; import com.google.auto.value.AutoValue; @@ -59,6 +57,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.logging.Level; +import java.util.logging.LogRecord; import java.util.logging.Logger; import org.jspecify.annotations.Nullable; @@ -126,67 +125,31 @@ static synchronized CloudDatastoreV1ClientImpl create(DatastoreServiceConfig con @Override public Future beginTransaction(final BeginTransactionRequest req) { - return makeCall( - new Callable() { - @Override - public BeginTransactionResponse call() throws DatastoreException { - return datastore.beginTransaction(req); - } - }); + return makeCall(() -> datastore.beginTransaction(req)); } @Override public Future rollback(final RollbackRequest req) { - return makeCall( - new Callable() { - @Override - public RollbackResponse call() throws DatastoreException { - return datastore.rollback(req); - } - }); + return makeCall(() -> datastore.rollback(req)); } @Override public Future runQuery(final RunQueryRequest req) { - return makeCall( - new Callable() { - @Override - public RunQueryResponse call() throws DatastoreException { - return datastore.runQuery(req); - } - }); + return makeCall(() -> datastore.runQuery(req)); } @Override public Future lookup(final LookupRequest req) { - return makeCall( - new Callable() { - @Override - public LookupResponse call() throws DatastoreException { - return datastore.lookup(req); - } - }); + return makeCall(() -> datastore.lookup(req)); } @Override public Future allocateIds(final AllocateIdsRequest req) { - return makeCall( - new Callable() { - @Override - public AllocateIdsResponse call() throws DatastoreException { - return datastore.allocateIds(req); - } - }); + return makeCall(() -> datastore.allocateIds(req)); } private Future commit(final CommitRequest req) { - return makeCall( - new Callable() { - @Override - public CommitResponse call() throws DatastoreException { - return datastore.commit(req); - } - }); + return makeCall(() -> datastore.commit(req)); } @Override @@ -222,10 +185,11 @@ public T call() throws Exception { return callable.call(); } catch (Exception e) { if (isRetryable(e) && remainingTries > 0) { - logger.log( - Level.FINE, - String.format("Caught retryable exception; %d tries remaining", remainingTries), - e); + LogRecord lr = + new LogRecord(Level.FINE, "Caught retryable exception; {0} tries remaining"); + lr.setParameters(new Object[] {remainingTries}); + lr.setThrown(e); + logger.log(lr); Thread.sleep(backoff.nextBackOffMillis()); } else { throw e; @@ -249,21 +213,17 @@ private Future makeCall(final Callable oneAttempt) { ? new Exception() : null; return executor.submit( - new Callable() { - @Override - public T call() throws Exception { - try { - return new RetryingCallable<>(oneAttempt, maxRetries).call(); - } catch (DatastoreException e) { - String message = - stackTraceCapturer != null - ? String.format( - "%s%nstack trace when async call was initiated: <%n%s>", - e.getMessage(), Throwables.getStackTraceAsString(stackTraceCapturer)) - : String.format( - "%s%n(stack trace capture for async call is disabled)", e.getMessage()); - throw DatastoreApiHelper.createV1Exception(e.getCode(), message, e); - } + () -> { + try { + return new RetryingCallable<>(oneAttempt, maxRetries).call(); + } catch (DatastoreException e) { + String message = + stackTraceCapturer != null + ? e.getMessage() + + "\nstack trace when async call was initiated: <\n" + + Throwables.getStackTraceAsString(stackTraceCapturer) + : e.getMessage() + "\n(stack trace capture for async call is disabled)"; + throw DatastoreApiHelper.createV1Exception(e.getCode(), message, e); } }); } @@ -275,13 +235,10 @@ private static DatastoreOptions createDatastoreOptions( setProjectEndpoint(projectId, options); options.credential(getCredential()); options.initializer( - new HttpRequestInitializer() { - @Override - public void initialize(HttpRequest request) throws IOException { - request.setConnectTimeout(httpConnectTimeoutMillis); - if (config.getDeadline() != null) { - request.setReadTimeout((int) (config.getDeadline() * 1000)); - } + request -> { + request.setConnectTimeout(httpConnectTimeoutMillis); + if (config.getDeadline() != null) { + request.setReadTimeout((int) (config.getDeadline() * 1000)); } }); return options.build(); @@ -298,8 +255,7 @@ private static Credential getCredential() throws GeneralSecurityException, IOExc if (privateKeyFile != null) { logger.log( Level.INFO, - "Service account and private key file were provided. " - + "Using service account credential."); + "Service account and private key file were provided. Using service account credential."); return getServiceAccountCredentialBuilder(serviceAccount) .setServiceAccountPrivateKeyFromP12File(new File(privateKeyFile)) .build(); @@ -308,8 +264,7 @@ private static Credential getCredential() throws GeneralSecurityException, IOExc if (privateKey != null) { logger.log( Level.INFO, - "Service account and private key were provided. " - + "Using service account credential."); + "Service account and private key were provided. Using service account credential."); return getServiceAccountCredentialBuilder(serviceAccount) .setServiceAccountPrivateKey(privateKey) .build(); diff --git a/api/src/main/java/com/google/appengine/api/datastore/CompositeIndexManager.java b/api/src/main/java/com/google/appengine/api/datastore/CompositeIndexManager.java index 27cdc2c82..d837718cb 100644 --- a/api/src/main/java/com/google/appengine/api/datastore/CompositeIndexManager.java +++ b/api/src/main/java/com/google/appengine/api/datastore/CompositeIndexManager.java @@ -145,12 +145,7 @@ protected String generateXmlForIndex(Index index, IndexSource source) { /** We compare {@link Property Properties} by comparing their names. */ private static final Comparator PROPERTY_NAME_COMPARATOR = - new Comparator() { - @Override - public int compare(Property o1, Property o2) { - return o1.getName().compareTo(o2.getName()); - } - }; + (o1, o2) -> o1.getName().compareTo(o2.getName()); private List getRecommendedIndexProps(IndexComponentsOnlyQuery query) { // Construct the list of index properties diff --git a/api/src/main/java/com/google/appengine/api/datastore/DataTypeTranslator.java b/api/src/main/java/com/google/appengine/api/datastore/DataTypeTranslator.java index ab7c174a9..0634623e2 100644 --- a/api/src/main/java/com/google/appengine/api/datastore/DataTypeTranslator.java +++ b/api/src/main/java/com/google/appengine/api/datastore/DataTypeTranslator.java @@ -1113,7 +1113,6 @@ public RawValue getValue(Value propertyValue) { } @Override - @ SuppressWarnings("PatternMatchingInstanceof") public @Nullable Comparable asComparable(Object value) { Object value2 = ((RawValue) value).getValue(); // All possible values except byte[] are already comparable. @@ -2027,10 +2026,7 @@ public int compareTo(ComparableByteArray other) { @Override public boolean equals(@Nullable Object obj) { - if (obj == null) { - return false; - } - return Arrays.equals(bytes, ((ComparableByteArray) obj).bytes); + return obj instanceof ComparableByteArray other && Arrays.equals(bytes, other.bytes); } @Override diff --git a/api/src/main/java/com/google/appengine/api/datastore/DatastoreApiHelper.java b/api/src/main/java/com/google/appengine/api/datastore/DatastoreApiHelper.java index 20087d628..6e880fca3 100644 --- a/api/src/main/java/com/google/appengine/api/datastore/DatastoreApiHelper.java +++ b/api/src/main/java/com/google/appengine/api/datastore/DatastoreApiHelper.java @@ -63,59 +63,41 @@ public static RuntimeException translateError(ApiProxy.ApplicationException exce if (errorCode == null) { return new DatastoreFailureException(exception.getErrorDetail()); } - switch (errorCode) { - case BAD_REQUEST: - return new IllegalArgumentException(exception.getErrorDetail()); - - case CONCURRENT_TRANSACTION: - return new ConcurrentModificationException(exception.getErrorDetail()); - - case NEED_INDEX: - return new DatastoreNeedIndexException(exception.getErrorDetail()); - - case TIMEOUT: - case BIGTABLE_ERROR: - return new DatastoreTimeoutException(exception.getErrorDetail()); - - case COMMITTED_BUT_STILL_APPLYING: - return new CommittedButStillApplyingException(exception.getErrorDetail()); - - case RESOURCE_EXHAUSTED: - return new ApiProxy.OverQuotaException(exception.getErrorDetail(), (Throwable) null); - - case INTERNAL_ERROR: - default: - return new DatastoreFailureException(exception.getErrorDetail()); - } + return switch (errorCode) { + case BAD_REQUEST -> new IllegalArgumentException(exception.getErrorDetail()); + case CONCURRENT_TRANSACTION -> new ConcurrentModificationException(exception.getErrorDetail()); + case NEED_INDEX -> new DatastoreNeedIndexException(exception.getErrorDetail()); + case TIMEOUT, BIGTABLE_ERROR -> new DatastoreTimeoutException(exception.getErrorDetail()); + case COMMITTED_BUT_STILL_APPLYING -> new CommittedButStillApplyingException( + exception.getErrorDetail()); + case RESOURCE_EXHAUSTED -> new ApiProxy.OverQuotaException( + exception.getErrorDetail(), (Throwable) null); + default -> new DatastoreFailureException(exception.getErrorDetail()); + }; } static RuntimeException createV1Exception(Code code, String message, Throwable cause) { if (code == null) { return new DatastoreFailureException(message, cause); } - switch (code) { - case ABORTED: - return new ConcurrentModificationException(message, cause); - case FAILED_PRECONDITION: + return switch (code) { + case ABORTED -> new ConcurrentModificationException(message, cause); + case FAILED_PRECONDITION -> { if (message.contains("The Cloud Datastore API is not enabled for the project")) { - return new DatastoreFailureException(message, cause); + yield new DatastoreFailureException(message, cause); } // Could also indicate ErrorCode.SAFE_TIME_TOO_OLD. - return new DatastoreNeedIndexException(message, cause); - case DEADLINE_EXCEEDED: - return new DatastoreTimeoutException(message, cause); - case INVALID_ARGUMENT: - case PERMISSION_DENIED: - return new IllegalArgumentException(message, cause); - case UNAVAILABLE: - return new ApiProxy.RPCFailedException(message, cause); - case RESOURCE_EXHAUSTED: - return new ApiProxy.OverQuotaException(message, cause); - case INTERNAL: - // Could also indicate ErrorCode.COMMITTED_BUT_STILL_APPLYING. - default: - return new DatastoreFailureException(message, cause); - } + yield new DatastoreNeedIndexException(message, cause); + } + case DEADLINE_EXCEEDED -> new DatastoreTimeoutException(message, cause); + case INVALID_ARGUMENT, PERMISSION_DENIED -> new IllegalArgumentException(message, cause); + case UNAVAILABLE -> new ApiProxy.RPCFailedException(message, cause); + case RESOURCE_EXHAUSTED -> new ApiProxy.OverQuotaException(message, cause); + case INTERNAL -> + // Could also indicate ErrorCode.COMMITTED_BUT_STILL_APPLYING. + new DatastoreFailureException(message, cause); + default -> new DatastoreFailureException(message, cause); + }; } static Future makeAsyncCall( @@ -162,8 +144,8 @@ protected T wrap(byte[] responseBytes) throws InvalidProtocolBufferException { @Override protected Throwable convertException(Throwable cause) { - if (cause instanceof ApiProxy.ApplicationException) { - return translateError((ApiProxy.ApplicationException) cause); + if (cause instanceof ApiProxy.ApplicationException applicationException) { + return translateError(applicationException); } return cause; } diff --git a/api/src/main/java/com/google/appengine/api/datastore/DatastoreCallbacksImpl.java b/api/src/main/java/com/google/appengine/api/datastore/DatastoreCallbacksImpl.java index 39c79f0cf..2575d9736 100644 --- a/api/src/main/java/com/google/appengine/api/datastore/DatastoreCallbacksImpl.java +++ b/api/src/main/java/com/google/appengine/api/datastore/DatastoreCallbacksImpl.java @@ -336,22 +336,19 @@ private static Object newInstance(Class cls) { } private Callback allocateCallback(final Object callbackImplementor, final Method callbackMethod) { - return new Callback() { - @Override - public void run(CallbackContext context) { - try { - callbackMethod.invoke(callbackImplementor, context); - } catch (IllegalAccessException e) { - // shouldn't happen since we made the method accessible - throw new RuntimeException(e); - } catch (InvocationTargetException e) { - // TODO: Check whether it is possible for this to be null - if (e.getCause() != null) { - throwIfUnchecked(e.getCause()); - } - // Should not happen since we don't allow checked exceptions. - throw new RuntimeException("Callback method threw a checked exception.", e.getCause()); + return context -> { + try { + callbackMethod.invoke(callbackImplementor, context); + } catch (IllegalAccessException e) { + // shouldn't happen since we made the method accessible + throw new RuntimeException(e); + } catch (InvocationTargetException e) { + // TODO: Check whether it is possible for this to be null + if (e.getCause() != null) { + throwIfUnchecked(e.getCause()); } + // Should not happen since we don't allow checked exceptions. + throw new RuntimeException("Callback method threw a checked exception.", e.getCause()); } }; } diff --git a/api/src/main/java/com/google/appengine/api/datastore/DatastoreConfig.java b/api/src/main/java/com/google/appengine/api/datastore/DatastoreConfig.java index 551416e49..7228ff393 100644 --- a/api/src/main/java/com/google/appengine/api/datastore/DatastoreConfig.java +++ b/api/src/main/java/com/google/appengine/api/datastore/DatastoreConfig.java @@ -39,10 +39,5 @@ public interface DatastoreConfig { */ @Deprecated DatastoreConfig DEFAULT = - new DatastoreConfig() { - @Override - public ImplicitTransactionManagementPolicy getImplicitTransactionManagementPolicy() { - return ImplicitTransactionManagementPolicy.NONE; - } - }; + () -> ImplicitTransactionManagementPolicy.NONE; } diff --git a/api/src/main/java/com/google/appengine/api/datastore/Entity.java b/api/src/main/java/com/google/appengine/api/datastore/Entity.java index 47add6ed5..783e435fe 100644 --- a/api/src/main/java/com/google/appengine/api/datastore/Entity.java +++ b/api/src/main/java/com/google/appengine/api/datastore/Entity.java @@ -94,8 +94,7 @@ public boolean getForceIndexedEmbeddedEntity() { @Override public boolean equals(@Nullable Object that) { - if (that instanceof WrappedValueImpl) { - WrappedValueImpl wv = (WrappedValueImpl) that; + if (that instanceof WrappedValueImpl wv) { return ((value == null) ? wv.value == null : value.equals(wv.value)) && indexed == wv.indexed && forceIndexedEmbeddedEntity == wv.forceIndexedEmbeddedEntity; @@ -141,8 +140,7 @@ public boolean getForceIndexedEmbeddedEntity() { @Override public boolean equals(@Nullable Object that) { - if (that instanceof UnindexedValue) { - UnindexedValue uv = (UnindexedValue) that; + if (that instanceof UnindexedValue uv) { return (value == null) ? uv.value == null : value.equals(uv.value); } return false; @@ -267,8 +265,7 @@ public Entity(Key key) { */ @Override public boolean equals(@Nullable Object object) { - if (object instanceof Entity) { - Entity otherEntity = (Entity) object; + if (object instanceof Entity otherEntity) { return key.equals(otherEntity.key); } return false; diff --git a/api/src/main/java/com/google/appengine/api/datastore/FilterMatcher.java b/api/src/main/java/com/google/appengine/api/datastore/FilterMatcher.java index 8b58c2117..366b37a83 100644 --- a/api/src/main/java/com/google/appengine/api/datastore/FilterMatcher.java +++ b/api/src/main/java/com/google/appengine/api/datastore/FilterMatcher.java @@ -175,45 +175,39 @@ public boolean matches(List> values) { public void addFilter(Filter filter) { Comparable value = DataTypeTranslator.getComparablePropertyValue(filter.getProperty(0)); switch (filter.getOp()) { - case EQUAL: - equalValues.add(value); - break; - case GREATER_THAN: + case EQUAL -> equalValues.add(value); + case GREATER_THAN -> { if (min == NoValue.INSTANCE || EntityProtoComparators.MULTI_TYPE_COMPARATOR.compare(min, value) <= 0) { min = value; minInclusive = false; } - break; - case GREATER_THAN_OR_EQUAL: + } + case GREATER_THAN_OR_EQUAL -> { if (min == NoValue.INSTANCE || EntityProtoComparators.MULTI_TYPE_COMPARATOR.compare(min, value) < 0) { min = value; minInclusive = true; } - break; - case LESS_THAN: + } + case LESS_THAN -> { if (max == NoValue.INSTANCE || EntityProtoComparators.MULTI_TYPE_COMPARATOR.compare(max, value) >= 0) { max = value; maxInclusive = false; } - break; - case LESS_THAN_OR_EQUAL: + } + case LESS_THAN_OR_EQUAL -> { if (max == NoValue.INSTANCE || EntityProtoComparators.MULTI_TYPE_COMPARATOR.compare(max, value) > 0) { max = value; maxInclusive = true; } - break; - case EXISTS: - break; - case CONTAINED_IN_REGION: - geoRegions.add(fromProto(filter.getGeoRegion())); - break; - default: - throw new IllegalArgumentException( - "Unable to perform filter using operator " + filter.getOp()); + } + case EXISTS -> {} + case CONTAINED_IN_REGION -> geoRegions.add(fromProto(filter.getGeoRegion())); + default -> throw new IllegalArgumentException( + "Unable to perform filter using operator " + filter.getOp()); } } diff --git a/api/src/main/java/com/google/appengine/api/datastore/Index.java b/api/src/main/java/com/google/appengine/api/datastore/Index.java index a84451661..a8dfc12f5 100644 --- a/api/src/main/java/com/google/appengine/api/datastore/Index.java +++ b/api/src/main/java/com/google/appengine/api/datastore/Index.java @@ -85,8 +85,7 @@ public String getName() { // manually written @Override public boolean equals(@Nullable Object obj) { - if (obj instanceof Property) { - Property other = (Property) obj; + if (obj instanceof Property other) { return name.equals(other.name) && Objects.equals(direction, other.direction); } return false; @@ -158,8 +157,7 @@ public List getProperties() { @Override public boolean equals(@Nullable Object obj) { - if (obj instanceof Index) { - Index other = (Index) obj; + if (obj instanceof Index other) { return id == other.id && kind.equals(other.kind) && isAncestor == other.isAncestor diff --git a/api/src/main/java/com/google/appengine/api/datastore/Key.java b/api/src/main/java/com/google/appengine/api/datastore/Key.java index 35060c8a3..5185da3ad 100644 --- a/api/src/main/java/com/google/appengine/api/datastore/Key.java +++ b/api/src/main/java/com/google/appengine/api/datastore/Key.java @@ -288,8 +288,7 @@ public boolean equals(@Nullable Object object) { * @return if key is equal to this. */ boolean equals(@Nullable Object object, boolean considerNotAssigned) { - if (object instanceof Key) { - Key key = (Key) object; + if (object instanceof Key key) { if (this == key) { return true; } diff --git a/api/src/main/java/com/google/appengine/api/datastore/Link.java b/api/src/main/java/com/google/appengine/api/datastore/Link.java index 3903d3327..c53e97d8d 100644 --- a/api/src/main/java/com/google/appengine/api/datastore/Link.java +++ b/api/src/main/java/com/google/appengine/api/datastore/Link.java @@ -64,8 +64,7 @@ public int hashCode() { /** Two {@code Link} objects are considered equal if their content strings match exactly. */ @Override public boolean equals(@Nullable Object object) { - if (object instanceof Link) { - Link key = (Link) object; + if (object instanceof Link key) { return value.equals(key.value); } return false; diff --git a/api/src/main/java/com/google/appengine/api/datastore/PreparedMultiQuery.java b/api/src/main/java/com/google/appengine/api/datastore/PreparedMultiQuery.java index ac07288c0..b5c12d5c0 100644 --- a/api/src/main/java/com/google/appengine/api/datastore/PreparedMultiQuery.java +++ b/api/src/main/java/com/google/appengine/api/datastore/PreparedMultiQuery.java @@ -281,14 +281,7 @@ private Iterator makeQueryIterator() { // use a heap iterator to merge the results from multiple sources // this may not respect the limit passed to it in fetchOptions return makeHeapIterator( - Iterables.transform( - queries, - new Function>() { - @Override - public Iterator apply(PreparedQuery input) { - return input.asIterator(fetchOptions); - } - })); + Iterables.transform(queries, input -> input.asIterator(fetchOptions))); } } diff --git a/api/src/main/java/com/google/appengine/api/datastore/Query.java b/api/src/main/java/com/google/appengine/api/datastore/Query.java index 73b4aae44..fb0b4c744 100644 --- a/api/src/main/java/com/google/appengine/api/datastore/Query.java +++ b/api/src/main/java/com/google/appengine/api/datastore/Query.java @@ -574,12 +574,10 @@ public boolean equals(@Nullable Object o) { if (this == o) { return true; } - if (o == null || getClass() != o.getClass()) { + if (!(o instanceof Query query)) { return false; } - Query query = (Query) o; - if (keysOnly != query.keysOnly) { return false; } @@ -739,12 +737,10 @@ public boolean equals(@Nullable Object o) { if (this == o) { return true; } - if (o == null || getClass() != o.getClass()) { + if (!(o instanceof SortPredicate that)) { return false; } - SortPredicate that = (SortPredicate) o; - if (direction != that.direction) { return false; } @@ -864,10 +860,9 @@ public boolean equals(@Nullable Object obj) { if (this == obj) { return true; } - if (!(obj instanceof CompositeFilter)) { + if (!(obj instanceof CompositeFilter other)) { return false; } - CompositeFilter other = (CompositeFilter) obj; if (operator != other.operator) { return false; } @@ -942,12 +937,10 @@ public boolean equals(@Nullable Object o) { if (this == o) { return true; } - if (o == null || getClass() != o.getClass()) { + if (!(o instanceof FilterPredicate that)) { return false; } - FilterPredicate that = (FilterPredicate) o; - if (operator != that.operator) { return false; } @@ -1007,12 +1000,10 @@ public boolean equals(@Nullable Object o) { if (this == o) { return true; } - if (o == null || getClass() != o.getClass()) { + if (!(o instanceof StContainsFilter that)) { return false; } - StContainsFilter that = (StContainsFilter) o; - if (!propertyName.equals(that.propertyName)) { return false; } @@ -1034,7 +1025,7 @@ int hashCodeNoFilterValues() { @Override public String toString() { - return String.format("StContainsFilter [%s: %s]", propertyName, region); + return "StContainsFilter [%s: %s]".formatted(propertyName, region); } } @@ -1096,12 +1087,10 @@ public boolean equals(@Nullable Object o) { return true; } - if (o == null || getClass() != o.getClass()) { + if (!(o instanceof Circle other)) { return false; } - Circle other = (Circle) o; - if (!center.equals(other.center)) { return false; } @@ -1119,7 +1108,7 @@ public int hashCode() { @Override public String toString() { - return String.format("Circle [(%s),%f]", center, radius); + return "Circle [(%s),%f]".formatted(center, radius); } } @@ -1173,12 +1162,10 @@ public boolean equals(@Nullable Object o) { return true; } - if (o == null || getClass() != o.getClass()) { + if (!(o instanceof Rectangle other)) { return false; } - Rectangle other = (Rectangle) o; - if (!southwest.equals(other.southwest)) { return false; } @@ -1196,7 +1183,7 @@ public int hashCode() { @Override public String toString() { - return String.format("Rectangle [(%s),(%s)]", southwest, northeast); + return "Rectangle [(%s),(%s)]".formatted(southwest, northeast); } } } diff --git a/api/src/main/java/com/google/appengine/api/datastore/QueryTranslator.java b/api/src/main/java/com/google/appengine/api/datastore/QueryTranslator.java index 823309ef3..6f904c59b 100644 --- a/api/src/main/java/com/google/appengine/api/datastore/QueryTranslator.java +++ b/api/src/main/java/com/google/appengine/api/datastore/QueryTranslator.java @@ -143,14 +143,11 @@ static Order convertSortPredicateToPb(Query.SortPredicate predicate) { } private static Direction getSortOp(Query.SortDirection direction) { - switch (direction) { - case ASCENDING: - return Direction.ASCENDING; - case DESCENDING: - return Direction.DESCENDING; - default: - throw new UnsupportedOperationException("direction: " + direction); - } + return switch (direction) { + case ASCENDING -> Direction.ASCENDING; + case DESCENDING -> Direction.DESCENDING; + default -> throw new UnsupportedOperationException("direction: " + direction); + }; } /** @@ -159,16 +156,14 @@ private static Direction getSortOp(Query.SortDirection direction) { * validated, so we complete the validation here. */ private static void copyGeoFilterToPb(Query.Filter filter, DatastoreV3Pb.Query.Builder proto) { - if (filter instanceof Query.CompositeFilter) { - Query.CompositeFilter conjunction = (Query.CompositeFilter) filter; + if (filter instanceof Query.CompositeFilter conjunction) { checkArgument( conjunction.getOperator() == Query.CompositeFilterOperator.AND, "Geo-spatial filters may only be composed with CompositeFilterOperator.AND"); for (Query.Filter f : conjunction.getSubFilters()) { copyGeoFilterToPb(f, proto); } - } else if (filter instanceof Query.StContainsFilter) { - Query.StContainsFilter containmentFilter = (Query.StContainsFilter) filter; + } else if (filter instanceof Query.StContainsFilter containmentFilter) { Filter.Builder f = proto.addFilterBuilder(); f.setOp(Operator.CONTAINED_IN_REGION); f.setGeoRegion(convertGeoRegionToPb(containmentFilter.getRegion())); @@ -180,13 +175,13 @@ private static void copyGeoFilterToPb(Query.Filter filter, DatastoreV3Pb.Query.B .setName(containmentFilter.getPropertyName()) .setMultiple(false) .setValue(PropertyValue.getDefaultInstance()); - } else { - checkArgument(filter instanceof Query.FilterPredicate); - Query.FilterPredicate predicate = (Query.FilterPredicate) filter; + } else if (filter instanceof Query.FilterPredicate predicate) { checkArgument( predicate.getOperator() == Query.FilterOperator.EQUAL, "Geo-spatial filters may only be combined with equality comparisons"); proto.addFilterBuilder().mergeFrom(convertFilterPredicateToPb(predicate)); + } else { + checkArgument(false); } } @@ -217,16 +212,14 @@ private static Filter convertFilterPredicateToPb(Query.FilterPredicate predicate private static DatastoreV3Pb.GeoRegion convertGeoRegionToPb(Query.GeoRegion region) { DatastoreV3Pb.GeoRegion.Builder geoRegion = DatastoreV3Pb.GeoRegion.newBuilder(); - if (region instanceof Query.GeoRegion.Circle) { - Query.GeoRegion.Circle circle = (Query.GeoRegion.Circle) region; + if (region instanceof Query.GeoRegion.Circle circle) { DatastoreV3Pb.CircleRegion circlePb = DatastoreV3Pb.CircleRegion.newBuilder() .setCenter(convertGeoPtToPb(circle.getCenter())) .setRadiusMeters(circle.getRadius()) .build(); geoRegion.setCircle(circlePb); - } else if (region instanceof Query.GeoRegion.Rectangle) { - Query.GeoRegion.Rectangle rect = (Query.GeoRegion.Rectangle) region; + } else if (region instanceof Query.GeoRegion.Rectangle rect) { DatastoreV3Pb.RectangleRegion rectPb = DatastoreV3Pb.RectangleRegion.newBuilder() .setSouthwest(convertGeoPtToPb(rect.getSouthwest())) @@ -247,22 +240,15 @@ private static DatastoreV3Pb.RegionPoint convertGeoPtToPb(GeoPt point) { } private static Operator getFilterOp(Query.FilterOperator operator) { - switch (operator) { - case LESS_THAN: - return Operator.LESS_THAN; - case LESS_THAN_OR_EQUAL: - return Operator.LESS_THAN_OR_EQUAL; - case GREATER_THAN: - return Operator.GREATER_THAN; - case GREATER_THAN_OR_EQUAL: - return Operator.GREATER_THAN_OR_EQUAL; - case EQUAL: - return Operator.EQUAL; - case IN: - return Operator.IN; - default: - throw new UnsupportedOperationException("operator: " + operator); - } + return switch (operator) { + case LESS_THAN -> Operator.LESS_THAN; + case LESS_THAN_OR_EQUAL -> Operator.LESS_THAN_OR_EQUAL; + case GREATER_THAN -> Operator.GREATER_THAN; + case GREATER_THAN_OR_EQUAL -> Operator.GREATER_THAN_OR_EQUAL; + case EQUAL -> Operator.EQUAL; + case IN -> Operator.IN; + default -> throw new UnsupportedOperationException("operator: " + operator); + }; } // All methods are static. Do not instantiate. diff --git a/api/src/main/java/com/google/appengine/api/datastore/RawValue.java b/api/src/main/java/com/google/appengine/api/datastore/RawValue.java index f196244ec..fb871cfa6 100644 --- a/api/src/main/java/com/google/appengine/api/datastore/RawValue.java +++ b/api/src/main/java/com/google/appengine/api/datastore/RawValue.java @@ -122,39 +122,35 @@ public final class RawValue implements Serializable { return asType(User.class); } } else if (valueV1 != null) { - switch (valueV1.getValueTypeCase()) { - case BOOLEAN_VALUE: - return valueV1.getBooleanValue(); - case DOUBLE_VALUE: - return valueV1.getDoubleValue(); - case INTEGER_VALUE: - return valueV1.getIntegerValue(); - case ENTITY_VALUE: + return switch (valueV1.getValueTypeCase()) { + case BOOLEAN_VALUE -> valueV1.getBooleanValue(); + case DOUBLE_VALUE -> valueV1.getDoubleValue(); + case INTEGER_VALUE -> valueV1.getIntegerValue(); + case ENTITY_VALUE -> { if (valueV1.getMeaning() == 20) { - return asType(User.class); + yield asType(User.class); } throw new IllegalStateException("Raw entity value is not supported."); - case KEY_VALUE: - return asType(Key.class); - case STRING_VALUE: - return valueV1.getStringValueBytes().toByteArray(); - case BLOB_VALUE: - // TODO: return a short blob? (not currently possible to get here). - return valueV1.getBlobValue().toByteArray(); - case TIMESTAMP_VALUE: - // TODO: return a Date? (not currently possible to get here). - return DatastoreHelper.getTimestamp(valueV1); - case GEO_POINT_VALUE: - if (valueV1.getMeaning() == 0 || valueV1.getMeaning() == Meaning.INDEX_VALUE.getNumber()) { - return asType(GeoPt.class); + } + case KEY_VALUE -> asType(Key.class); + case STRING_VALUE -> valueV1.getStringValueBytes().toByteArray(); + case BLOB_VALUE -> + // TODO: return a short blob? (not currently possible to get here). + valueV1.getBlobValue().toByteArray(); + case TIMESTAMP_VALUE -> + // TODO: return a Date? (not currently possible to get here). + DatastoreHelper.getTimestamp(valueV1); + case GEO_POINT_VALUE -> { + if (valueV1.getMeaning() == 0 + || valueV1.getMeaning() == Meaning.INDEX_VALUE.getNumber()) { + yield asType(GeoPt.class); } - break; // GeoPt with meaning becomes null. - case ARRAY_VALUE: - throw new IllegalStateException("Raw array value is not supported."); - case NULL_VALUE: - default: - return null; - } + yield null; // GeoPt with meaning becomes null. + } + case ARRAY_VALUE -> throw new IllegalStateException("Raw array value is not supported."); + case NULL_VALUE -> null; + default -> null; + }; } return null; } diff --git a/api/src/main/java/com/google/appengine/api/datastore/ShortBlob.java b/api/src/main/java/com/google/appengine/api/datastore/ShortBlob.java index 94f79e972..9ddaa5285 100644 --- a/api/src/main/java/com/google/appengine/api/datastore/ShortBlob.java +++ b/api/src/main/java/com/google/appengine/api/datastore/ShortBlob.java @@ -70,8 +70,7 @@ public int hashCode() { /** Two {@code ShortBlob} objects are considered equal if their contained bytes match exactly. */ @Override public boolean equals(@Nullable Object object) { - if (object instanceof ShortBlob) { - ShortBlob other = (ShortBlob) object; + if (object instanceof ShortBlob other) { return Arrays.equals(bytes, other.bytes); } return false; diff --git a/api/src/main/java/com/google/appengine/api/datastore/Text.java b/api/src/main/java/com/google/appengine/api/datastore/Text.java index f5cde6a14..f2a6907dc 100644 --- a/api/src/main/java/com/google/appengine/api/datastore/Text.java +++ b/api/src/main/java/com/google/appengine/api/datastore/Text.java @@ -69,8 +69,7 @@ public int hashCode() { /** Two {@code Text} objects are considered equal if their content strings match exactly. */ @Override public boolean equals(@Nullable Object object) { - if (object instanceof Text) { - Text key = (Text) object; + if (object instanceof Text key) { if (value == null) { return key.value == null; } diff --git a/api/src/main/java/com/google/appengine/api/datastore/TransactionImpl.java b/api/src/main/java/com/google/appengine/api/datastore/TransactionImpl.java index 8c753bf8b..36890a8a0 100644 --- a/api/src/main/java/com/google/appengine/api/datastore/TransactionImpl.java +++ b/api/src/main/java/com/google/appengine/api/datastore/TransactionImpl.java @@ -122,8 +122,8 @@ public String getId() { @Override public boolean equals(@Nullable Object o) { - if (o instanceof TransactionImpl) { - return internalTransaction.equals(((TransactionImpl) o).internalTransaction); + if (o instanceof TransactionImpl that) { + return internalTransaction.equals(that.internalTransaction); } return false; } diff --git a/api/src/main/java/com/google/appengine/api/images/ImagesServiceImpl.java b/api/src/main/java/com/google/appengine/api/images/ImagesServiceImpl.java index 1d447b78d..dc402dd80 100644 --- a/api/src/main/java/com/google/appengine/api/images/ImagesServiceImpl.java +++ b/api/src/main/java/com/google/appengine/api/images/ImagesServiceImpl.java @@ -140,8 +140,8 @@ protected Image wrap(byte @Nullable[] responseBytes) throws IOException { @Override protected Throwable convertException(Throwable cause) { - if (cause instanceof ApiProxy.ApplicationException) { - return convertApplicationException(request, (ApiProxy.ApplicationException) cause); + if (cause instanceof ApiProxy.ApplicationException applicationException) { + return convertApplicationException(request, applicationException); } return cause; } @@ -400,38 +400,32 @@ private ImagesTransformRequest.Builder generateImagesTransformRequest( private ImagesServicePb.OutputSettings convertOutputSettings(OutputSettings settings) { ImagesServicePb.OutputSettings.Builder pbSettings = ImagesServicePb.OutputSettings.newBuilder(); - switch(settings.getOutputEncoding()) { - case PNG: - pbSettings.setMimeType(MIME_TYPE.PNG); - break; - case JPEG: + switch (settings.getOutputEncoding()) { + case PNG -> pbSettings.setMimeType(MIME_TYPE.PNG); + case JPEG -> { pbSettings.setMimeType(MIME_TYPE.JPEG); if (settings.hasQuality()) { pbSettings.setQuality(settings.getQuality()); } - break; - case WEBP: + } + case WEBP -> { pbSettings.setMimeType(MIME_TYPE.WEBP); if (settings.hasQuality()) { pbSettings.setQuality(settings.getQuality()); } - break; - default: - throw new IllegalArgumentException( - "Invalid output encoding requested"); + } + default -> throw new IllegalArgumentException("Invalid output encoding requested"); } return pbSettings.build(); } private ImagesServicePb.InputSettings convertInputSettings(InputSettings settings) { ImagesServicePb.InputSettings.Builder pbSettings = ImagesServicePb.InputSettings.newBuilder(); - switch(settings.getOrientationCorrection()) { - case UNCHANGED_ORIENTATION: - pbSettings.setCorrectExifOrientation(ORIENTATION_CORRECTION_TYPE.UNCHANGED_ORIENTATION); - break; - case CORRECT_ORIENTATION: - pbSettings.setCorrectExifOrientation(ORIENTATION_CORRECTION_TYPE.CORRECT_ORIENTATION); - break; + switch (settings.getOrientationCorrection()) { + case UNCHANGED_ORIENTATION -> + pbSettings.setCorrectExifOrientation(ORIENTATION_CORRECTION_TYPE.UNCHANGED_ORIENTATION); + case CORRECT_ORIENTATION -> + pbSettings.setCorrectExifOrientation(ORIENTATION_CORRECTION_TYPE.CORRECT_ORIENTATION); } return pbSettings.build(); } diff --git a/api/src/main/java/com/google/appengine/api/mail/MailServiceImpl.java b/api/src/main/java/com/google/appengine/api/mail/MailServiceImpl.java index 55700f61b..18d4560e8 100644 --- a/api/src/main/java/com/google/appengine/api/mail/MailServiceImpl.java +++ b/api/src/main/java/com/google/appengine/api/mail/MailServiceImpl.java @@ -120,21 +120,15 @@ private void doSend(Message message, boolean toAdmin) } catch (ApiProxy.ApplicationException ex) { // Pass all the error details straight through (same as python). switch (ErrorCode.forNumber(ex.getApplicationError())) { - case BAD_REQUEST: - throw new IllegalArgumentException("Bad Request: " + - ex.getErrorDetail()); - case UNAUTHORIZED_SENDER: - throw new IllegalArgumentException("Unauthorized Sender: " + - ex.getErrorDetail()); - case INVALID_ATTACHMENT_TYPE: - throw new IllegalArgumentException("Invalid Attachment Type: " + - ex.getErrorDetail()); - case INVALID_HEADER_NAME: - throw new IllegalArgumentException("Invalid Header Name: " + - ex.getErrorDetail()); - case INTERNAL_ERROR: - default: - throw new IOException(ex.getErrorDetail()); + case BAD_REQUEST -> throw new IllegalArgumentException("Bad Request: " + + ex.getErrorDetail()); + case UNAUTHORIZED_SENDER -> throw new IllegalArgumentException("Unauthorized Sender: " + + ex.getErrorDetail()); + case INVALID_ATTACHMENT_TYPE -> throw new IllegalArgumentException("Invalid Attachment Type: " + + ex.getErrorDetail()); + case INVALID_HEADER_NAME -> throw new IllegalArgumentException("Invalid Header Name: " + + ex.getErrorDetail()); + default -> throw new IOException(ex.getErrorDetail()); } } } diff --git a/api/src/main/java/com/google/appengine/api/memcache/AsyncMemcacheServiceImpl.java b/api/src/main/java/com/google/appengine/api/memcache/AsyncMemcacheServiceImpl.java index f54e9775f..df2bf8b2d 100644 --- a/api/src/main/java/com/google/appengine/api/memcache/AsyncMemcacheServiceImpl.java +++ b/api/src/main/java/com/google/appengine/api/memcache/AsyncMemcacheServiceImpl.java @@ -234,35 +234,15 @@ public int hashCode() { private static class DefaultValueProviders { @SuppressWarnings("rawtypes") - private static final Provider NULL_PROVIDER = new Provider() { - @Override public Object get() { - return null; - } - }; + private static final Provider NULL_PROVIDER = () -> null; - private static final Provider FALSE_PROVIDER = new Provider() { - @Override public Boolean get() { - return Boolean.FALSE; - } - }; + private static final Provider FALSE_PROVIDER = () -> Boolean.FALSE; @SuppressWarnings("rawtypes") - private static final Provider SET_PROVIDER = - new Provider>() { - @Override - public Set get() { - return new HashSet<>(0, 1); - } - }; + private static final Provider SET_PROVIDER = () -> new HashSet<>(0, 1); @SuppressWarnings("rawtypes") - private static final Provider MAP_PROVIDER = - new Provider>() { - @Override - public Map get() { - return new HashMap<>(0, 1); - } - }; + private static final Provider MAP_PROVIDER = () -> new HashMap<>(0, 1); private static final Provider STATS_PROVIDER = new Provider() { final Stats emptyStats = new AsyncMemcacheServiceImpl.StatsImpl(null); diff --git a/api/src/main/java/com/google/appengine/api/memcache/MemcacheServiceImpl.java b/api/src/main/java/com/google/appengine/api/memcache/MemcacheServiceImpl.java index fe7ee3bb3..531ae830d 100644 --- a/api/src/main/java/com/google/appengine/api/memcache/MemcacheServiceImpl.java +++ b/api/src/main/java/com/google/appengine/api/memcache/MemcacheServiceImpl.java @@ -43,10 +43,10 @@ private static T quietGet(Future future) { throw new MemcacheServiceException("Unexpected failure", e); } catch (ExecutionException e) { Throwable cause = e.getCause(); - if (cause instanceof RuntimeException) { - throw (RuntimeException) cause; - } else if (cause instanceof Error) { - throw (Error) cause; + if (cause instanceof RuntimeException runtimeException) { + throw runtimeException; + } else if (cause instanceof Error error) { + throw error; } else { throw new UndeclaredThrowableException(cause); } diff --git a/api/src/main/java/com/google/appengine/api/search/Document.java b/api/src/main/java/com/google/appengine/api/search/Document.java index 2db68bd4a..10d0b4a2d 100644 --- a/api/src/main/java/com/google/appengine/api/search/Document.java +++ b/api/src/main/java/com/google/appengine/api/search/Document.java @@ -22,6 +22,7 @@ import com.google.apphosting.api.search.DocumentPb.Document.OrderIdSource; import com.google.common.base.Preconditions; import com.google.common.collect.HashMultimap; +import com.google.common.collect.Iterables; import com.google.common.collect.SetMultimap; import java.io.IOException; import java.io.ObjectInputStream; @@ -64,10 +65,10 @@ * * for (Field field : document.getFields()) { * switch (field.getType()) { - * case TEXT: use(field.getText()); break; - * case HTML: use(field.getHtml()); break; - * case ATOM: use(field.getAtom()); break; - * case DATE: use(field.getDate()); break; + * case TEXT -> use(field.getText()); + * case HTML -> use(field.getHtml()); + * case ATOM -> use(field.getAtom()); + * case DATE -> use(field.getDate()); * } * } * } @@ -78,8 +79,8 @@ * * for (Facet facet : document.getFacets()) { * switch (facet.getType()) { - * case ATOM: use(facet.getAtom()); break; - * case NUMBER: use(facet.getNumber()); break; + * case ATOM -> use(facet.getAtom()); + * case NUMBER -> use(facet.getNumber()); * } * } * } @@ -419,11 +420,10 @@ public boolean equals(Object object) { if (this == object) { return true; } - if (!(object instanceof Document)) { - return false; + if (object instanceof Document doc) { + return documentId.equals(doc.getId()); } - Document doc = (Document) object; - return documentId.equals(doc.getId()); + return false; } /** @@ -517,6 +517,7 @@ static Builder newBuilder(DocumentPb.Document document) { * @throws IllegalArgumentException if any parts of the document are invalid * or the document protocol buffer is too large */ + @SuppressWarnings("UnsafeLocaleUsage") DocumentPb.Document copyToProtocolBuffer() { DocumentPb.Document.Builder docBuilder = DocumentPb.Document.newBuilder(); if (documentId != null) { @@ -573,7 +574,7 @@ boolean isIdenticalTo(Document other) { return false; } } - if (!getFacets().equals(other.getFacets())) { + if (!Iterables.elementsEqual(getFacets(), other.getFacets())) { return false; } if (locale == null) { diff --git a/api/src/main/java/com/google/appengine/api/search/Facet.java b/api/src/main/java/com/google/appengine/api/search/Facet.java index 5723d101b..5ea451133 100644 --- a/api/src/main/java/com/google/appengine/api/search/Facet.java +++ b/api/src/main/java/com/google/appengine/api/search/Facet.java @@ -109,13 +109,12 @@ public boolean equals(Object object) { if (object == this) { return true; } - if (!(object instanceof Facet)) { - return false; + if (object instanceof Facet facet) { + return Util.equalObjects(name, facet.name) + && Util.equalObjects(atom, facet.atom) + && Util.equalObjects(number, facet.number); } - Facet facet = (Facet) object; - return Util.equalObjects(name, facet.name) - && Util.equalObjects(atom, facet.atom) - && Util.equalObjects(number, facet.number); + return false; } /** @@ -130,14 +129,14 @@ private void checkValid() { if (atom != null) { if (number != null) { throw new IllegalArgumentException( - String.format("both atom and number are set for facet %s", name)); + "both atom and number are set for facet %s".formatted(name)); } FacetChecker.checkAtom(atom); } else if (number != null) { FacetChecker.checkNumber(number); } else { throw new IllegalArgumentException( - String.format("neither atom nor number is set for facet %s", name)); + "neither atom nor number is set for facet %s".formatted(name)); } } diff --git a/api/src/main/java/com/google/appengine/api/search/Field.java b/api/src/main/java/com/google/appengine/api/search/Field.java index 05227c70e..e9cbd595a 100644 --- a/api/src/main/java/com/google/appengine/api/search/Field.java +++ b/api/src/main/java/com/google/appengine/api/search/Field.java @@ -446,11 +446,10 @@ public boolean equals(Object object) { if (object == this) { return true; } - if (!(object instanceof Field)) { - return false; + if (object instanceof Field field) { + return Util.equalObjects(name, field.name); } - Field field = (Field) object; - return Util.equalObjects(name, field.name); + return false; } /** @@ -463,6 +462,7 @@ public boolean equals(Object object) { * @throws IllegalArgumentException if field name, text, HTML, atom, * date, untokenizedPrefix, tokenizedPrefix, or vector are invalid */ + @SuppressWarnings("UnnecessaryDefaultInEnumSwitch") private Field checkValid() { FieldChecker.checkFieldName(name); if (type != null) { @@ -498,6 +498,7 @@ public static Builder newBuilder() { * @throws SearchException if the field contains invalid name, text, html, * atom, date */ + @SuppressWarnings("UnnecessaryDefaultInEnumSwitch") static Builder newBuilder(DocumentPb.Field field) { FieldValue value = field.getValue(); Field.Builder fieldBuilder = @@ -506,47 +507,32 @@ static Builder newBuilder(DocumentPb.Field field) { fieldBuilder.setLocale(FieldChecker.parseLocale(value.getLanguage())); } switch (value.getType()) { - case TEXT: - fieldBuilder.setText(value.getStringValue()); - break; - case HTML: - fieldBuilder.setHTML(value.getStringValue()); - break; - case ATOM: - fieldBuilder.setAtom(value.getStringValue()); - break; - case NUMBER: + case TEXT -> fieldBuilder.setText(value.getStringValue()); + case HTML -> fieldBuilder.setHTML(value.getStringValue()); + case ATOM -> fieldBuilder.setAtom(value.getStringValue()); + case NUMBER -> { try { fieldBuilder.setNumber( NumberFormat.getNumberInstance().parse(value.getStringValue()).doubleValue()); } catch (ParseException e) { throw new SearchException("Failed to parse double: " + value.getStringValue()); } - break; - case GEO: - fieldBuilder.setGeoPoint(GeoPoint.newGeoPoint(value.getGeo())); - break; - case DATE: + } + case GEO -> fieldBuilder.setGeoPoint(GeoPoint.newGeoPoint(value.getGeo())); + case DATE -> { String dateString = value.getStringValue(); if (dateString.isEmpty()) { throw new SearchException( - String.format("date not specified for field %s", field.getName())); + "date not specified for field %s".formatted(field.getName())); } fieldBuilder.setDate(DateUtil.deserializeDate(dateString)); - break; - case UNTOKENIZED_PREFIX: - fieldBuilder.setUntokenizedPrefix(value.getStringValue()); - break; - case TOKENIZED_PREFIX: - fieldBuilder.setTokenizedPrefix(value.getStringValue()); - break; - case VECTOR: - fieldBuilder.setVector(value.getVectorValueList()); - break; - default: - throw new SearchException( - String.format("unknown field value type %s for field %s", value.getType(), - field.getName())); + } + case UNTOKENIZED_PREFIX -> fieldBuilder.setUntokenizedPrefix(value.getStringValue()); + case TOKENIZED_PREFIX -> fieldBuilder.setTokenizedPrefix(value.getStringValue()); + case VECTOR -> fieldBuilder.setVector(value.getVectorValueList()); + default -> throw new SearchException( + "unknown field value type %s for field %s".formatted(value.getType(), + field.getName())); } return fieldBuilder; } @@ -558,6 +544,7 @@ static Builder newBuilder(DocumentPb.Field field) { * @return the field protocol buffer copy of this field object * @throws IllegalArgumentException if the field value type is unknown */ + @SuppressWarnings("UnnecessaryDefaultInEnumSwitch") DocumentPb.Field copyToProtocolBuffer() { DocumentPb.FieldValue.Builder fieldValueBuilder = DocumentPb.FieldValue.newBuilder(); if (locale != null) { @@ -565,29 +552,29 @@ DocumentPb.Field copyToProtocolBuffer() { } if (type != null) { switch (type) { - case TEXT: + case TEXT -> { if (text != null) { fieldValueBuilder.setStringValue(text); } fieldValueBuilder.setType(ContentType.TEXT); - break; - case HTML: + } + case HTML -> { if (html != null) { fieldValueBuilder.setStringValue(html); } fieldValueBuilder.setType(ContentType.HTML); - break; - case ATOM: + } + case ATOM -> { if (atom != null) { fieldValueBuilder.setStringValue(atom); } fieldValueBuilder.setType(ContentType.ATOM); - break; - case DATE: + } + case DATE -> { fieldValueBuilder.setStringValue(DateUtil.serializeDate(date)); fieldValueBuilder.setType(ContentType.DATE); - break; - case NUMBER: + } + case NUMBER -> { // TODO: use binary number representation instead DecimalFormat format = new DecimalFormat(); format.setDecimalSeparatorAlwaysShown(false); @@ -595,29 +582,28 @@ DocumentPb.Field copyToProtocolBuffer() { format.setMaximumFractionDigits(Integer.MAX_VALUE); fieldValueBuilder.setStringValue(format.format(number)); fieldValueBuilder.setType(ContentType.NUMBER); - break; - case GEO_POINT: + } + case GEO_POINT -> { fieldValueBuilder.setGeo(geoPoint.copyToProtocolBuffer()); fieldValueBuilder.setType(ContentType.GEO); - break; - case UNTOKENIZED_PREFIX: + } + case UNTOKENIZED_PREFIX -> { if (untokenizedPrefix != null) { fieldValueBuilder.setStringValue(untokenizedPrefix); } fieldValueBuilder.setType(ContentType.UNTOKENIZED_PREFIX); - break; - case TOKENIZED_PREFIX: + } + case TOKENIZED_PREFIX -> { if (tokenizedPrefix != null) { fieldValueBuilder.setStringValue(tokenizedPrefix); } fieldValueBuilder.setType(ContentType.TOKENIZED_PREFIX); - break; - case VECTOR: + } + case VECTOR -> { fieldValueBuilder.addAllVectorValue(vector); fieldValueBuilder.setType(ContentType.VECTOR); - break; - default: - throw new IllegalArgumentException(String.format("unknown field type %s", type)); + } + default -> throw new IllegalArgumentException("unknown field type %s".formatted(type)); } } @@ -637,34 +623,27 @@ public String toString() { .finish(); } + @SuppressWarnings("UnnecessaryDefaultInEnumSwitch") private String valueToString() throws IllegalArgumentException { if (type == null) { return "null"; } - switch (type) { - case TEXT: - return text; - case HTML: - return html; - case ATOM: - return atom; - case DATE: - return DateUtil.formatDateTime(date); - case GEO_POINT: - return geoPoint.toString(); - case NUMBER: + return switch (type) { + case TEXT -> text; + case HTML -> html; + case ATOM -> atom; + case DATE -> DateUtil.formatDateTime(date); + case GEO_POINT -> geoPoint.toString(); + case NUMBER -> { DecimalFormat format = new DecimalFormat(); format.setDecimalSeparatorAlwaysShown(false); format.setMaximumFractionDigits(Integer.MAX_VALUE); - return format.format(number); - case UNTOKENIZED_PREFIX: - return untokenizedPrefix; - case TOKENIZED_PREFIX: - return tokenizedPrefix; - case VECTOR: - return vector.toString(); - default: - throw new IllegalArgumentException(String.format("unknown field type %s", type)); - } + yield format.format(number); + } + case UNTOKENIZED_PREFIX -> untokenizedPrefix; + case TOKENIZED_PREFIX -> tokenizedPrefix; + case VECTOR -> vector.toString(); + default -> throw new IllegalArgumentException("unknown field type %s".formatted(type)); + }; } } diff --git a/api/src/main/java/com/google/appengine/api/search/IndexImpl.java b/api/src/main/java/com/google/appengine/api/search/IndexImpl.java index 46e4d562a..f878bcba2 100644 --- a/api/src/main/java/com/google/appengine/api/search/IndexImpl.java +++ b/api/src/main/java/com/google/appengine/api/search/IndexImpl.java @@ -131,10 +131,9 @@ public boolean equals(Object obj) { if (this == obj) { return true; } - if (!(obj instanceof IndexImpl)) { + if (!(obj instanceof IndexImpl other)) { return false; } - IndexImpl other = (IndexImpl) obj; return Util.equalObjects(spec, other.spec) && Util.equalObjects(schema, other.schema) && Util.equalObjects(storageUsage, other.storageUsage) @@ -146,8 +145,8 @@ public String toString() { String storageInfo = (storageUsage == null || storageLimit == null) ? "(no storage data)" - : String.format(" (%d/%d)", storageUsage.longValue(), storageLimit.longValue()); - return String.format("IndexImpl{namespace: %s, %s, %s%s}", config.getNamespace(), spec, + : " (%d/%d)".formatted(storageUsage.longValue(), storageLimit.longValue()); + return "IndexImpl{namespace: %s, %s, %s%s}".formatted(config.getNamespace(), spec, (schema == null ? "(null schema)" : schema), storageInfo); } @@ -180,7 +179,7 @@ protected Void wrap(SearchServicePb.DeleteSchemaResponse.Builder key) throw new DeleteException( new OperationResult( StatusCode.INTERNAL_ERROR, - String.format("Expected 1 removed schema, but got %d", + "Expected 1 removed schema, but got %d".formatted( response.getStatusList().size())), results); } @@ -213,7 +212,7 @@ public Future deleteAsync(final Iterable documentIds) { } if (size > SearchApiLimits.PUT_MAXIMUM_DOCS_PER_REQUEST) { throw new IllegalArgumentException( - String.format("number of doc ids, %s, exceeds maximum %s", size, + "number of doc ids, %s, exceeds maximum %s".formatted(size, SearchApiLimits.PUT_MAXIMUM_DOCS_PER_REQUEST)); } final int documentIdsSize = size; @@ -240,7 +239,7 @@ protected Void wrap(SearchServicePb.DeleteDocumentResponse.Builder key) throw new DeleteException( new OperationResult( StatusCode.INTERNAL_ERROR, - String.format("Expected %d removed documents, but got %d", documentIdsSize, + "Expected %d removed documents, but got %d".formatted(documentIdsSize, response.getStatusList().size())), results); } @@ -291,8 +290,7 @@ public Future putAsync(final Iterable documents) { // add the current one. if (!document.isIdenticalTo(other)) { throw new IllegalArgumentException( - String.format( - "Put request with documents with the same ID \"%s\" but different content", + "Put request with documents with the same ID \"%s\" but different content".formatted( document.getId())); } } @@ -304,7 +302,7 @@ public Future putAsync(final Iterable documents) { } if (size > SearchApiLimits.PUT_MAXIMUM_DOCS_PER_REQUEST) { throw new IllegalArgumentException( - String.format("number of documents, %s, exceeds maximum %s", size, + "number of documents, %s, exceeds maximum %s".formatted(size, SearchApiLimits.PUT_MAXIMUM_DOCS_PER_REQUEST)); } final int documentsSize = size; @@ -327,7 +325,7 @@ protected PutResponse wrap(SearchServicePb.IndexDocumentResponse.Builder key) throw new PutException( new OperationResult( StatusCode.INTERNAL_ERROR, - String.format("Expected %d indexed documents, but got %d", documentsSize, + "Expected %d indexed documents, but got %d".formatted(documentsSize, response.getStatusList().size())), results, response.getDocIdList()); } for (OperationResult result : results) { diff --git a/api/src/main/java/com/google/appengine/api/search/checkers/FieldChecker.java b/api/src/main/java/com/google/appengine/api/search/checkers/FieldChecker.java index 1b3df8b31..c4e5552fe 100644 --- a/api/src/main/java/com/google/appengine/api/search/checkers/FieldChecker.java +++ b/api/src/main/java/com/google/appengine/api/search/checkers/FieldChecker.java @@ -24,6 +24,7 @@ import com.google.common.base.Preconditions; import com.google.common.base.Strings; import java.util.Date; +import java.util.IllformedLocaleException; import java.util.List; import java.util.Locale; import org.antlr.runtime.RecognitionException; @@ -237,8 +238,8 @@ private static String checkExpressionHelper(String expression, String mode) { try { parser.parse(expression); } catch (RecognitionException e) { - String message = String.format("Failed to parse %s expression '%s': " - + "parse error at line %d position %d", + String message = ("Failed to parse %s expression '%s': " + + "parse error at line %d position %d").formatted( mode, expression, e.line, e.charPositionInLine); throw new IllegalArgumentException(message); } @@ -269,37 +270,20 @@ public static String checkSortExpression(String expression) { return checkExpressionHelper(expression, "sort"); } + @SuppressWarnings("UnnecessaryDefaultInEnumSwitch") public static DocumentPb.Field checkValid(DocumentPb.Field field) { checkFieldName(field.getName()); DocumentPb.FieldValue value = field.getValue(); switch (value.getType()) { - case TEXT: - checkText(value.getStringValue()); - break; - case HTML: - checkHTML(value.getStringValue()); - break; - case DATE: - checkDate(DateUtil.deserializeDate(value.getStringValue())); - break; - case ATOM: - checkAtom(value.getStringValue()); - break; - case NUMBER: - checkNumber(Double.parseDouble(value.getStringValue())); - break; - case GEO: - GeoPointChecker.checkValid(value.getGeo()); - break; - case UNTOKENIZED_PREFIX: - case TOKENIZED_PREFIX: - checkPrefix(value.getStringValue()); - break; - case VECTOR: - checkVector(value.getVectorValueList()); - break; - default: - throw new IllegalArgumentException("Unsupported field type " + value.getType()); + case TEXT -> checkText(value.getStringValue()); + case HTML -> checkHTML(value.getStringValue()); + case DATE -> checkDate(DateUtil.deserializeDate(value.getStringValue())); + case ATOM -> checkAtom(value.getStringValue()); + case NUMBER -> checkNumber(Double.parseDouble(value.getStringValue())); + case GEO -> GeoPointChecker.checkValid(value.getGeo()); + case UNTOKENIZED_PREFIX, TOKENIZED_PREFIX -> checkPrefix(value.getStringValue()); + case VECTOR -> checkVector(value.getVectorValueList()); + default -> throw new IllegalArgumentException("Unsupported field type " + value.getType()); } return field; } @@ -319,15 +303,19 @@ public static Locale parseLocale(String locale) { // has more than 2 separators, e.g. "en-US-variant-with-hyphen", // we want 3 parts: "en", "US", "variant-with-hyphen". String[] parts = locale.split("[-_]", 3); - if (parts.length == 1) { - return new Locale(parts[0]); - } - if (parts.length == 2) { - return new Locale(parts[0], parts[1]); - } - if (parts.length == 3) { - return new Locale(parts[0], parts[1], parts[2]); + try { + return switch (parts.length) { + case 1 -> Locale.forLanguageTag(parts[0].replace('_', '-')); + case 2 -> new Locale.Builder().setLanguage(parts[0]).setRegion(parts[1]).build(); + case 3 -> new Locale.Builder() + .setLanguage(parts[0]) + .setRegion(parts[1]) + .setVariant(parts[2]) + .build(); + default -> throw new IllegalArgumentException("Cannot parse locale " + locale); + }; + } catch (IllformedLocaleException e) { + throw new IllegalArgumentException("Cannot parse locale " + locale, e); } - throw new IllegalArgumentException("Cannot parse locale " + locale); } } diff --git a/api/src/main/java/com/google/appengine/api/taskqueue/QueueApiHelper.java b/api/src/main/java/com/google/appengine/api/taskqueue/QueueApiHelper.java index 74199552e..c5b46583b 100644 --- a/api/src/main/java/com/google/appengine/api/taskqueue/QueueApiHelper.java +++ b/api/src/main/java/com/google/appengine/api/taskqueue/QueueApiHelper.java @@ -66,8 +66,8 @@ Future makeAsyncCall( return new FutureWrapper(response) { @Override protected Throwable convertException(Throwable cause) { - if (cause instanceof ApiProxy.ApplicationException) { - return translateError((ApiProxy.ApplicationException) cause); + if (cause instanceof ApiProxy.ApplicationException appException) { + return translateError(appException); } return cause; } @@ -105,10 +105,10 @@ static T getInternal(Future future) { ErrorCode.TRANSIENT_ERROR_VALUE, "Interrupted while waiting for RPC response."); } catch (ExecutionException e) { Throwable cause = e.getCause(); - if (cause instanceof RuntimeException) { - throw (RuntimeException) cause; - } else if (cause instanceof Error) { - throw (Error) cause; + if (cause instanceof RuntimeException runtimeException) { + throw runtimeException; + } else if (cause instanceof Error error) { + throw error; } else { throw new RuntimeException(cause); } @@ -133,66 +133,52 @@ static RuntimeException translateError(int error, String detail) { return taskqueueException; } - switch (errorCode) { - case UNKNOWN_QUEUE: - return new IllegalStateException("The specified queue is unknown : " + detail); - case TRANSIENT_ERROR: - return new TransientFailureException(detail); - case INTERNAL_ERROR: - return new InternalFailureException(detail); - case TASK_TOO_LARGE: - return new IllegalArgumentException("Task size is too large : " + detail); - case INVALID_TASK_NAME: - return new IllegalArgumentException("Invalid task name : " + detail); - case INVALID_QUEUE_NAME: - return new IllegalArgumentException("Invalid queue name : " + detail); - case INVALID_URL: - return new IllegalArgumentException("Invalid URL : " + detail); - case INVALID_QUEUE_RATE: - return new IllegalArgumentException("Invalid queue rate : " + detail); - case PERMISSION_DENIED: - return new SecurityException("Permission for requested operation is denied : " + detail); - case TASK_ALREADY_EXISTS: - return new TaskAlreadyExistsException("Task name already exists : " + detail); - case TOMBSTONED_TASK: - // TODO It may be more interesting to throw a - // "TombstonedTaskException" instead, however the semantics are currently - // not useful. (i.e. Race condition between attempt to execute task - // and when state of task is changed to TOMBSTONED_TASK means that - // TASK_ALREADY_EXISTS does not indicate that the task will be - // guaranteed to execute after the add request is initiated. This guarantee - // is required if TOMBSTONED_TASK is a useful state.) - // It may be nice to have 3 states (TaskInRunQueue, TaskRunning, TaskRetired). - // TaskInRunQueue and TaskRunning indicate the current TASK_ALREADY_EXISTS - // state. - return new TaskAlreadyExistsException("Task name is tombstoned : " + detail); - case INVALID_ETA: - return new IllegalArgumentException("ETA is invalid : " + detail); - case INVALID_REQUEST: - return new IllegalArgumentException("Invalid request : " + detail); - case UNKNOWN_TASK: - return new TaskNotFoundException("Task does not exist : " + detail); - case TOMBSTONED_QUEUE: - // TODO: Return a different exception when the Java API is able to provoke this - // exception e.g. if the ability to delete queues is added. - return new IllegalStateException( - "The queue has been marked for deletion and is no longer usable : " + detail); - case DUPLICATE_TASK_NAME: - return new IllegalArgumentException("Identical task names in request : " + detail); - // SKIPPED will never be translated into an exception. - case TOO_MANY_TASKS: - return new IllegalArgumentException("Request contains too many tasks : " + detail); - case INVALID_QUEUE_MODE: - return new InvalidQueueModeException( - "Target queue mode does not support this operation : " + detail); - case TASK_LEASE_EXPIRED: - return new IllegalStateException("The task lease has expired : " + detail); - case QUEUE_PAUSED: - return new IllegalStateException( - "The queue is paused and cannot process the request : " + detail); - default: - return new QueueFailureException("Unspecified error (" + errorCode + ") : " + detail); - } + return switch (errorCode) { + case UNKNOWN_QUEUE -> new IllegalStateException( + "The specified queue is unknown : " + detail); + case TRANSIENT_ERROR -> new TransientFailureException(detail); + case INTERNAL_ERROR -> new InternalFailureException(detail); + case TASK_TOO_LARGE -> new IllegalArgumentException("Task size is too large : " + detail); + case INVALID_TASK_NAME -> new IllegalArgumentException("Invalid task name : " + detail); + case INVALID_QUEUE_NAME -> new IllegalArgumentException("Invalid queue name : " + detail); + case INVALID_URL -> new IllegalArgumentException("Invalid URL : " + detail); + case INVALID_QUEUE_RATE -> new IllegalArgumentException("Invalid queue rate : " + detail); + case PERMISSION_DENIED -> new SecurityException( + "Permission for requested operation is denied : " + detail); + case TASK_ALREADY_EXISTS -> new TaskAlreadyExistsException( + "Task name already exists : " + detail); + case TOMBSTONED_TASK -> + // TODO It may be more interesting to throw a + // "TombstonedTaskException" instead, however the semantics are currently + // not useful. (i.e. Race condition between attempt to execute task + // and when state of task is changed to TOMBSTONED_TASK means that + // TASK_ALREADY_EXISTS does not indicate that the task will be + // guaranteed to execute after the add request is initiated. This guarantee + // is required if TOMBSTONED_TASK is a useful state.) + // It may be nice to have 3 states (TaskInRunQueue, TaskRunning, TaskRetired). + // TaskInRunQueue and TaskRunning indicate the current TASK_ALREADY_EXISTS + // state. + new TaskAlreadyExistsException("Task name is tombstoned : " + detail); + case INVALID_ETA -> new IllegalArgumentException("ETA is invalid : " + detail); + case INVALID_REQUEST -> new IllegalArgumentException("Invalid request : " + detail); + case UNKNOWN_TASK -> new TaskNotFoundException("Task does not exist : " + detail); + case TOMBSTONED_QUEUE -> + // TODO: Return a different exception when the Java API is able to provoke this + // exception e.g. if the ability to delete queues is added. + new IllegalStateException( + "The queue has been marked for deletion and is no longer usable : " + detail); + case DUPLICATE_TASK_NAME -> new IllegalArgumentException( + "Identical task names in request : " + detail); + // SKIPPED will never be translated into an exception. + case TOO_MANY_TASKS -> new IllegalArgumentException( + "Request contains too many tasks : " + detail); + case INVALID_QUEUE_MODE -> new InvalidQueueModeException( + "Target queue mode does not support this operation : " + detail); + case TASK_LEASE_EXPIRED -> new IllegalStateException("The task lease has expired : " + detail); + case QUEUE_PAUSED -> new IllegalStateException( + "The queue is paused and cannot process the request : " + detail); + default -> new QueueFailureException("Unspecified error (" + errorCode + ") : " + detail); + }; } static RuntimeException translateError(ApiProxy.ApplicationException exception) { diff --git a/api/src/main/java/com/google/appengine/api/taskqueue/QueueImpl.java b/api/src/main/java/com/google/appengine/api/taskqueue/QueueImpl.java index 7af43429d..59e3b6fa1 100644 --- a/api/src/main/java/com/google/appengine/api/taskqueue/QueueImpl.java +++ b/api/src/main/java/com/google/appengine/api/taskqueue/QueueImpl.java @@ -517,15 +517,13 @@ public Future> addAsync(Transaction txn, Iterable if (option.getTaskName() != null && !option.getTaskName().isEmpty()) { if (!taskNames.add(option.getTaskName())) { throw new IllegalArgumentException( - String.format( - "Identical task names in request : \"%s\" duplicated", option.getTaskName())); + "Identical task names in request : \"%s\" duplicated".formatted(option.getTaskName())); } } } if (bulkAddRequest.getAddRequestCount() > QueueConstants.maxTasksPerAdd()) { throw new IllegalArgumentException( - String.format( - "No more than %d tasks can be added in a single add call", + "No more than %d tasks can be added in a single add call".formatted( QueueConstants.maxTasksPerAdd())); } @@ -537,8 +535,7 @@ public Future> addAsync(Transaction txn, Iterable if (txn != null && builtRequest.getSerializedSize() > QueueConstants.maxTransactionalRequestSizeBytes()) { throw new IllegalArgumentException( - String.format( - "Transactional add may not be larger than %d bytes: %d bytes requested.", + "Transactional add may not be larger than %d bytes: %d bytes requested.".formatted( QueueConstants.maxTransactionalRequestSizeBytes(), builtRequest.getSerializedSize())); } @@ -549,8 +546,7 @@ public Future> addAsync(Transaction txn, Iterable protected List wrap(TaskQueueBulkAddResponse bulkAddResponse) { if (bulkAddResponse.getTaskResultCount() != bulkAddRequest.getAddRequestCount()) { throw new InternalFailureException( - String.format( - "expected %d results from BulkAdd(), got %d", + "expected %d results from BulkAdd(), got %d".formatted( bulkAddRequest.getAddRequestCount(), bulkAddResponse.getTaskResultCount())); } @@ -593,8 +589,9 @@ protected List wrap(TaskQueueBulkAddResponse bulkAddResponse) { if (taskqueueException == null) { taskqueueException = e; } - TaskAlreadyExistsException taee = (TaskAlreadyExistsException) taskqueueException; - taee.appendTaskName(options.getTaskName()); + if (taskqueueException instanceof TaskAlreadyExistsException taee) { + taee.appendTaskName(options.getTaskName()); + } } else { taskqueueException = e; } @@ -756,10 +753,8 @@ public Future> deleteTaskAsync(List taskHandles) { deleteRequest.addTaskName(ByteString.copyFromUtf8(taskHandle.getName())); } else { throw new QueueNameMismatchException( - String.format( - "The task %s is associated with the queue named %s " - + "and cannot be deleted from the queue named %s.", - taskHandle.getName(), taskHandle.getQueueName(), this.queueName)); + "The task %s is associated with the queue named %s and cannot be deleted from the queue named %s." + .formatted(taskHandle.getName(), taskHandle.getQueueName(), this.queueName)); } } @@ -791,14 +786,14 @@ private Future> leaseTasksInternal(LeaseOptions options) { long leaseMillis = options.getUnit().toMillis(options.getLease()); if (leaseMillis > QueueConstants.maxLease(MILLISECONDS)) { throw new IllegalArgumentException( - String.format( - "A lease period can be no longer than %d seconds", QueueConstants.maxLease(SECONDS))); + "A lease period can be no longer than %d seconds".formatted( + QueueConstants.maxLease(SECONDS))); } if (options.getCountLimit() > QueueConstants.maxLeaseCount()) { throw new IllegalArgumentException( - String.format( - "No more than %d tasks can be leased in one call", QueueConstants.maxLeaseCount())); + "No more than %d tasks can be leased in one call".formatted( + QueueConstants.maxLeaseCount())); } TaskQueueQueryAndOwnTasksRequest.Builder leaseRequest = @@ -925,15 +920,13 @@ public TaskHandle modifyTaskLease(TaskHandle taskHandle, long lease, TimeUnit un long leaseMillis = unit.toMillis(lease); if (leaseMillis > QueueConstants.maxLease(MILLISECONDS)) { throw new IllegalArgumentException( - String.format( - "The lease time specified (%s seconds) is too large. " - + "Lease period can be no longer than %d seconds.", - formatLeaseTimeInSeconds(leaseMillis), QueueConstants.maxLease(SECONDS))); + "The lease time specified (%s seconds) is too large. Lease period can be no longer than %d seconds." + .formatted( + formatLeaseTimeInSeconds(leaseMillis), QueueConstants.maxLease(SECONDS))); } if (leaseMillis < 0) { throw new IllegalArgumentException( - String.format( - "The lease time must not be negative. Specified lease time was %s seconds.", + "The lease time must not be negative. Specified lease time was %s seconds.".formatted( formatLeaseTimeInSeconds(leaseMillis))); } @@ -956,7 +949,7 @@ private String formatLeaseTimeInSeconds(long milliSeconds) { long seconds = TimeUnit.SECONDS.convert(milliSeconds, TimeUnit.MILLISECONDS); long remainder = milliSeconds - TimeUnit.MILLISECONDS.convert(seconds, TimeUnit.SECONDS); String formatString = milliSeconds < 0 ? "-%01d.%03d" : "%01d.%03d"; - return String.format(formatString, Math.abs(seconds), Math.abs(remainder)); + return formatString.formatted(Math.abs(seconds), Math.abs(remainder)); } /** See {@link Queue#fetchStatistics()}. */ diff --git a/api/src/main/java/com/google/appengine/api/taskqueue/TaskOptions.java b/api/src/main/java/com/google/appengine/api/taskqueue/TaskOptions.java index fd006e5a5..74e84cc18 100644 --- a/api/src/main/java/com/google/appengine/api/taskqueue/TaskOptions.java +++ b/api/src/main/java/com/google/appengine/api/taskqueue/TaskOptions.java @@ -162,13 +162,10 @@ public boolean equals(Object o) { return true; } - if (!(o instanceof StringValueParam)) { - return false; + if (o instanceof StringValueParam that) { + return value.equals(that.value) && name.equals(that.name); } - - StringValueParam that = (StringValueParam) o; - - return value.equals(that.value) && name.equals(that.name); + return false; } @Override @@ -202,13 +199,10 @@ public boolean equals(Object o) { return true; } - if (!(o instanceof ByteArrayValueParam)) { - return false; + if (o instanceof ByteArrayValueParam that) { + return Arrays.equals(value, that.value) && name.equals(that.name); } - - ByteArrayValueParam that = (ByteArrayValueParam) o; - - return Arrays.equals(value, that.value) && name.equals(that.name); + return false; } /** @@ -469,11 +463,11 @@ public TaskOptions method(Method method) { public Map> getStringParams() { LinkedHashMap> stringParams = new LinkedHashMap<>(); for (Param param : params) { - if (param instanceof StringValueParam) { + if (param instanceof StringValueParam stringParam) { if (!stringParams.containsKey(param.name)) { stringParams.put(param.name, new ArrayList()); } - stringParams.get(param.name).add(((StringValueParam) param).value); + stringParams.get(param.name).add(stringParam.value); } } return stringParams; @@ -486,11 +480,11 @@ public Map> getStringParams() { public Map> getByteArrayParams() { LinkedHashMap> byteArrayParams = new LinkedHashMap<>(); for (Param param : params) { - if (param instanceof ByteArrayValueParam) { + if (param instanceof ByteArrayValueParam byteArrayParam) { if (!byteArrayParams.containsKey(param.name)) { byteArrayParams.put(param.name, new ArrayList()); } - byte[] value = ((ByteArrayValueParam) param).value; + byte[] value = byteArrayParam.value; byteArrayParams.get(param.name).add(Arrays.copyOf(value, value.length)); } } diff --git a/api/src/main/java/com/google/appengine/api/users/User.java b/api/src/main/java/com/google/appengine/api/users/User.java index 7646ea78a..cd290b85b 100644 --- a/api/src/main/java/com/google/appengine/api/users/User.java +++ b/api/src/main/java/com/google/appengine/api/users/User.java @@ -171,15 +171,13 @@ public boolean equals(@Nullable Object object) { // We're intentionally ignoring userId here because it is not // guaranteed to be present. This is consistent with the Python // API. - if (!(object instanceof User)) { - return false; + if (object instanceof User user) { + if ((federatedIdentity != null) && !federatedIdentity.isEmpty()) { + return user.federatedIdentity.equals(federatedIdentity); + } + return user.email.equals(email) && user.authDomain.equals(authDomain); } - - User user = (User) object; - if ((federatedIdentity != null) && (!federatedIdentity.isEmpty())) { - return user.federatedIdentity.equals(federatedIdentity); - } - return user.email.equals(email) && user.authDomain.equals(authDomain); + return false; } @Override diff --git a/api/src/main/java/com/google/appengine/api/users/UserServiceImpl.java b/api/src/main/java/com/google/appengine/api/users/UserServiceImpl.java index d9f2f368c..bc485b042 100644 --- a/api/src/main/java/com/google/appengine/api/users/UserServiceImpl.java +++ b/api/src/main/java/com/google/appengine/api/users/UserServiceImpl.java @@ -160,19 +160,17 @@ private byte[] makeSyncCall( } catch (ApiProxy.ApplicationException ex) { UserServiceError.ErrorCode errorCode = UserServiceError.ErrorCode.forNumber(ex.getApplicationError()); - switch (errorCode) { - case REDIRECT_URL_TOO_LONG: - throw new IllegalArgumentException("URL too long: " + destinationURL); - case NOT_ALLOWED: - throw new IllegalArgumentException( - "The requested URL was not allowed: " + destinationURL); - default: - // the python stub just rethrows, but I don't think we should be - // letting ApiProxy.ApplicationException propagate above the api - // layer. Also, having an api-specific failure exception is consistent with - // the datastore api. - throw new UserServiceFailureException(ex.getErrorDetail()); - } + throw switch (errorCode) { + case REDIRECT_URL_TOO_LONG -> new IllegalArgumentException("URL too long: " + destinationURL); + case NOT_ALLOWED -> new IllegalArgumentException( + "The requested URL was not allowed: " + destinationURL); + default -> + // the python stub just rethrows, but I don't think we should be + // letting ApiProxy.ApplicationException propagate above the api + // layer. Also, having an api-specific failure exception is consistent with + // the datastore api. + new UserServiceFailureException(ex.getErrorDetail()); + }; } return responseBytes; diff --git a/api/src/main/java/com/google/appengine/api/utils/SystemProperty.java b/api/src/main/java/com/google/appengine/api/utils/SystemProperty.java index 87952a044..0c5f5d918 100644 --- a/api/src/main/java/com/google/appengine/api/utils/SystemProperty.java +++ b/api/src/main/java/com/google/appengine/api/utils/SystemProperty.java @@ -42,6 +42,7 @@ public class SystemProperty { * {@code "com.google.appengine.runtime.environment"}. * Has the values {@code "Production"} and {@code "Development"}. */ + @SuppressWarnings("ClassInitializationDeadlock") public static final Environment environment = new Environment(); /** diff --git a/api/src/main/java/com/google/appengine/setup/RequestThreadFactory.java b/api/src/main/java/com/google/appengine/setup/RequestThreadFactory.java index 205b5c40b..9a07cf274 100644 --- a/api/src/main/java/com/google/appengine/setup/RequestThreadFactory.java +++ b/api/src/main/java/com/google/appengine/setup/RequestThreadFactory.java @@ -61,17 +61,14 @@ public RequestThreadFactory(Environment requestEnvironment) { public Thread newThread(final Runnable runnable) { checkState(requestEnvironment != null, "Request threads can only be created within the context of a running request."); - Thread thread = new Thread(new Runnable() { - @Override - public void run() { - if (runnable == null) { - return; - } - checkState(allowNewRequestThreadCreation, - "Cannot start new threads after the request thread stops."); - ApiProxy.setEnvironmentForCurrentThread(requestEnvironment); - runnable.run(); + Thread thread = new Thread(() -> { + if (runnable == null) { + return; } + checkState(allowNewRequestThreadCreation, + "Cannot start new threads after the request thread stops."); + ApiProxy.setEnvironmentForCurrentThread(requestEnvironment); + runnable.run(); }); checkState( allowNewRequestThreadCreation, "Cannot create new threads after the request thread stops."); diff --git a/api_dev/src/main/java/com/google/appengine/api/images/dev/LocalImagesService.java b/api_dev/src/main/java/com/google/appengine/api/images/dev/LocalImagesService.java index 203f01fb6..a5b56ed43 100644 --- a/api_dev/src/main/java/com/google/appengine/api/images/dev/LocalImagesService.java +++ b/api_dev/src/main/java/com/google/appengine/api/images/dev/LocalImagesService.java @@ -65,6 +65,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.logging.Level; import java.util.logging.Logger; import javax.imageio.ImageIO; import javax.imageio.ImageReader; @@ -124,18 +125,20 @@ public void init(LocalServiceContext context, Map properties) { String[] outputFormats = {"png", "jpg", "webp"}; for (String format : inputFormats) { if (!ImageIO.getImageReadersByFormatName(format).hasNext()) { - log.warning( - "No image reader found for format \"" + format + "\"." - + " An ImageIO plugin must be installed to use this format" - + " with the DevAppServer."); + log.log( + Level.WARNING, + "No image reader found for format \"{0}\". An ImageIO plugin must be installed to use" + + " this format with the DevAppServer.", + format); } } for (String format : outputFormats) { if (!ImageIO.getImageWritersByFormatName(format).hasNext()) { - log.warning( - "No image writer found for format \"" + format + "\"." - + " An ImageIO plugin must be installed to use this format" - + " with the DevAppServer."); + log.log( + Level.WARNING, + "No image writer found for format \"{0}\". An ImageIO plugin must be installed to use" + + " this format with the DevAppServer.", + format); } } @@ -344,8 +347,8 @@ Exif getExifMetadata(ImageData imageData) { ErrorCode.NOT_IMAGE.getNumber(), "Failed to read image EXIF metadata"); } AbstractImageInfo info = transform.getImageInfo(); - if (info instanceof Exif) { - return (Exif) info; + if (info instanceof Exif exif) { + return exif; } } catch (IOException ex) { throw new ApiProxy.ApplicationException( @@ -472,8 +475,7 @@ public ImagesHistogramResponse histogram( public ImagesGetUrlBaseResponse getUrlBase( final Status status, final ImagesGetUrlBaseRequest request) { if (request.getCreateSecureUrl()) { - log.info( - "Secure URLs will not be created using the development " + "application server."); + log.info("Secure URLs will not be created using the development application server."); } // Detect the image mimetype to see if is a valid image. ImageData imageData = @@ -518,25 +520,22 @@ public Integer getMaxApiRequestSize() { BufferedImage correctOrientation(BufferedImage image, Status status, int orientation) { Transform.Builder transform = Transform.newBuilder(); Transform.Builder secondTransform = Transform.newBuilder(); - switch(orientation) { - case 2: - return processTransform(image, transform.setHorizontalFlip(true).build(), status); - case 3: - return processTransform(image, transform.setRotate(180).build(), status); - case 4: - return processTransform(image, transform.setVerticalFlip(true).build(), status); - case 5: + return switch (orientation) { + case 2 -> processTransform(image, transform.setHorizontalFlip(true).build(), status); + case 3 -> processTransform(image, transform.setRotate(180).build(), status); + case 4 -> processTransform(image, transform.setVerticalFlip(true).build(), status); + case 5 -> { image = processTransform(image, transform.setVerticalFlip(true).build(), status); - return processTransform(image, secondTransform.setRotate(90).build(), status); - case 6: - return processTransform(image, transform.setRotate(90).build(), status); - case 7: + yield processTransform(image, secondTransform.setRotate(90).build(), status); + } + case 6 -> processTransform(image, transform.setRotate(90).build(), status); + case 7 -> { image = processTransform(image, transform.setHorizontalFlip(true).build(), status); - return processTransform(image, secondTransform.setRotate(90).build(), status); - case 8: - return processTransform(image, transform.setRotate(270).build(), status); - } - return image; + yield processTransform(image, secondTransform.setRotate(90).build(), status); + } + case 8 -> processTransform(image, transform.setRotate(270).build(), status); + default -> image; + }; } /** diff --git a/api_dev/src/main/java/com/google/appengine/api/search/dev/GenericScorer.java b/api_dev/src/main/java/com/google/appengine/api/search/dev/GenericScorer.java index 174b5f4dd..ed0c07a8a 100644 --- a/api_dev/src/main/java/com/google/appengine/api/search/dev/GenericScorer.java +++ b/api_dev/src/main/java/com/google/appengine/api/search/dev/GenericScorer.java @@ -116,9 +116,9 @@ public static Scorer newInstance( sorters.addAll(expression.getSorters( spec.getSortDescending() ? 1 : -1, numericDefault, spec.getDefaultValueText())); - if (expression instanceof NumericExpression) { + if (expression instanceof NumericExpression numericExpression) { expressions[i] = new NumericDefaultExpression( - (NumericExpression) expression, numericDefault); + numericExpression, numericDefault); } else { expressions[i] = new ExpressionBuilder.IntValueExpression(numericDefault); } diff --git a/api_dev/src/main/java/com/google/appengine/api/search/dev/PrefixFieldAnalyzerUtil.java b/api_dev/src/main/java/com/google/appengine/api/search/dev/PrefixFieldAnalyzerUtil.java index d2990e92f..4ab54a7a3 100644 --- a/api_dev/src/main/java/com/google/appengine/api/search/dev/PrefixFieldAnalyzerUtil.java +++ b/api_dev/src/main/java/com/google/appengine/api/search/dev/PrefixFieldAnalyzerUtil.java @@ -25,6 +25,7 @@ import java.util.ArrayList; import java.util.LinkedList; import java.util.List; +import java.util.Locale; import java.util.logging.Level; import java.util.logging.Logger; import org.apache.lucene.analysis.LetterTokenizer; @@ -43,7 +44,9 @@ final class PrefixFieldAnalyzerUtil { static String normalizePrefixField(String value) { String normalizedString = Normalizer.normalize(value, Normalizer.Form.NFKC); - return CharMatcher.whitespace().trimAndCollapseFrom(normalizedString, ' ').toLowerCase(); + return CharMatcher.whitespace() + .trimAndCollapseFrom(normalizedString, ' ') + .toLowerCase(Locale.ROOT); } static List createUntokenizedPrefixes(String value) { @@ -83,7 +86,7 @@ protected char normalize(char c) { */ @Override protected boolean isTokenChar(char c) { - return !LuceneUtils.WORD_SEPARATORS.contains(new Character(c)); + return !LuceneUtils.WORD_SEPARATORS.contains(Character.valueOf(c)); } } diff --git a/api_dev/src/main/java/com/google/appengine/tools/development/AbstractContainerService.java b/api_dev/src/main/java/com/google/appengine/tools/development/AbstractContainerService.java index 63e81ebfb..7c6bc95f5 100644 --- a/api_dev/src/main/java/com/google/appengine/tools/development/AbstractContainerService.java +++ b/api_dev/src/main/java/com/google/appengine/tools/development/AbstractContainerService.java @@ -36,7 +36,6 @@ import java.net.UnknownHostException; import java.security.Permissions; import java.util.Collection; -import java.util.Collections; import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.logging.Level; @@ -118,13 +117,7 @@ public abstract class AbstractContainerService implements ContainerService { // mapping is available. The advantage to this approach is that it avoids changing // the public ContainerService interface and hence avoids exposing our management // of port mappings to users. - protected PortMappingProvider portMappingProvider = - new PortMappingProvider() { - @Override - public Map getPortMapping() { - return Collections.emptyMap(); - } - }; + protected PortMappingProvider portMappingProvider = () -> ImmutableMap.of(); /** * Latch that will open once the module instance is fully initialized. TODO: This is used by some diff --git a/api_dev/src/main/java/com/google/appengine/tools/development/ApiProxyLocalImpl.java b/api_dev/src/main/java/com/google/appengine/tools/development/ApiProxyLocalImpl.java index db6343e8a..e59f01477 100644 --- a/api_dev/src/main/java/com/google/appengine/tools/development/ApiProxyLocalImpl.java +++ b/api_dev/src/main/java/com/google/appengine/tools/development/ApiProxyLocalImpl.java @@ -176,10 +176,10 @@ public byte[] makeSyncCall(ApiProxy.Environment environment, String packageName, } catch (CancellationException ex) { throw new ApiProxy.CancelledException(packageName, methodName); } catch (ExecutionException ex) { - if (ex.getCause() instanceof RuntimeException) { - throw (RuntimeException) ex.getCause(); - } else if (ex.getCause() instanceof Error) { - throw (Error) ex.getCause(); + if (ex.getCause() instanceof RuntimeException runtimeException) { + throw runtimeException; + } else if (ex.getCause() instanceof Error error) { + throw error; } else { throw new ApiProxy.UnknownException(packageName, methodName, ex.getCause()); } @@ -411,8 +411,8 @@ private byte[] callInternal() { } return invokeApiMethodJava(packageName, methodName, requestBytes); } catch (InvocationTargetException e) { - if (e.getCause() instanceof RuntimeException) { - throw (RuntimeException) e.getCause(); + if (e.getCause() instanceof RuntimeException runtimeException) { + throw runtimeException; } throw new UnknownException(packageName, methodName, e.getCause()); } catch (ReflectiveOperationException e) { diff --git a/api_dev/src/main/java/com/google/appengine/tools/development/ApiUtils.java b/api_dev/src/main/java/com/google/appengine/tools/development/ApiUtils.java index 613903c0e..08d67130f 100644 --- a/api_dev/src/main/java/com/google/appengine/tools/development/ApiUtils.java +++ b/api_dev/src/main/java/com/google/appengine/tools/development/ApiUtils.java @@ -93,8 +93,8 @@ public static Object convertBytesToPb(byte[] bytes, Class messageClass) * Message} (the open-sourced protocol buffer implementation). */ public static byte[] convertPbToBytes(Object object) { - if (object instanceof MessageLite) { - return ((MessageLite) object).toByteArray(); + if (object instanceof MessageLite messageLite) { + return messageLite.toByteArray(); } try { Class protocolMessageClass = Class.forName("com.google.io.protocol.ProtocolMessage"); diff --git a/api_dev/src/main/java/com/google/appengine/tools/development/Modules.java b/api_dev/src/main/java/com/google/appengine/tools/development/Modules.java index 9f74fab1c..fdeb6ec58 100644 --- a/api_dev/src/main/java/com/google/appengine/tools/development/Modules.java +++ b/api_dev/src/main/java/com/google/appengine/tools/development/Modules.java @@ -227,12 +227,7 @@ public String getScalingType(final String moduleName) throws ApplicationExceptio @Override public void startModule(final String moduleName, final String version) throws ApplicationException { - doDynamicConfiguration("startServing", new Runnable(){ - @Override - public void run() { - doStartModule(moduleName, version); - } - }); + doDynamicConfiguration("startServing", () -> doStartModule(moduleName, version)); } private void doStartModule(String moduleName, String version) { @@ -251,12 +246,7 @@ private void doStartModule(String moduleName, String version) { @Override public void stopModule(final String moduleName, final String version) throws ApplicationException { - doDynamicConfiguration("stopServing", new Runnable(){ - @Override - public void run() { - doStopModule(moduleName, version); - } - }); + doDynamicConfiguration("stopServing", () -> doStopModule(moduleName, version)); } /** diff --git a/api_dev/src/main/java/com/google/appengine/tools/development/RequestThreadFactory.java b/api_dev/src/main/java/com/google/appengine/tools/development/RequestThreadFactory.java index 28b3f966c..2974efc11 100644 --- a/api_dev/src/main/java/com/google/appengine/tools/development/RequestThreadFactory.java +++ b/api_dev/src/main/java/com/google/appengine/tools/development/RequestThreadFactory.java @@ -65,43 +65,42 @@ public synchronized void start() { super.start(); final Thread thread = this; // Thread.this doesn't work from an anon subclass RequestEndListenerHelper.register( - new RequestEndListener() { - @Override - public void onRequestEnd(ApiProxy.Environment environment) { + env -> { + if (thread.isAlive()) { + logger.info("Interrupting request thread: " + thread); + // This is one of the few places where it's okay to call thread.interrupt(). + @SuppressWarnings("Interruption") + boolean unused1 = true; + thread.interrupt(); + logger.info("Waiting up to 100ms for thread to complete: " + thread); + try { + thread.join(100); + } catch (InterruptedException ex) { + logger.info("Interrupted while waiting."); + } if (thread.isAlive()) { - logger.info("Interrupting request thread: " + thread); + logger.info("Interrupting request thread again: " + thread); + @SuppressWarnings("Interruption") + boolean unused2 = true; thread.interrupt(); - logger.info("Waiting up to 100ms for thread to complete: " + thread); + long remaining = getRemainingDeadlineMillis(env); + logger.info( + "Waiting up to " + remaining + " ms for thread to complete: " + thread); try { - thread.join(100); + thread.join(remaining); } catch (InterruptedException ex) { logger.info("Interrupted while waiting."); } if (thread.isAlive()) { - logger.info("Interrupting request thread again: " + thread); - thread.interrupt(); - long remaining = getRemainingDeadlineMillis(environment); - logger.info( - "Waiting up to " - + remaining - + " ms for thread to complete: " - + thread); - try { - thread.join(remaining); - } catch (InterruptedException ex) { - logger.info("Interrupted while waiting."); - } - if (thread.isAlive()) { - Throwable stack = new Throwable(); - stack.setStackTrace(thread.getStackTrace()); - logger.log( - Level.SEVERE, - "Thread left running: " - + thread - + ". " - + "In production this will cause the request to fail.", - stack); - } + Throwable stack = new Throwable(); + stack.setStackTrace(thread.getStackTrace()); + logger.log( + Level.SEVERE, + "Thread left running: " + + thread + + ". " + + "In production this will cause the request to fail.", + stack); } } } diff --git a/api_dev/src/main/java/com/google/appengine/tools/development/testing/LocalServiceTestHelper.java b/api_dev/src/main/java/com/google/appengine/tools/development/testing/LocalServiceTestHelper.java index 0d6268733..ee2916421 100644 --- a/api_dev/src/main/java/com/google/appengine/tools/development/testing/LocalServiceTestHelper.java +++ b/api_dev/src/main/java/com/google/appengine/tools/development/testing/LocalServiceTestHelper.java @@ -80,12 +80,7 @@ public interface RequestMillisTimer { * The Timer instance used by local services if no override is provided via * {@link LocalServiceTestHelper#setRemainingMillisTimer(RequestMillisTimer)}. */ - RequestMillisTimer DEFAULT = new RequestMillisTimer() { - @Override - public long getRemainingMillis() { - return Long.MAX_VALUE; - } - }; + RequestMillisTimer DEFAULT = () -> Long.MAX_VALUE; } // Keep these in sync with other occurrences of @@ -142,12 +137,12 @@ public LocalServiceTestHelper(LocalServiceTestConfig... configs) { ImmutableList.Builder builder = ImmutableList.builder(); LocalModulesServiceTestConfig configuredModulesServiceTestConfig = null; for (LocalServiceTestConfig config : configs) { - if (config instanceof LocalModulesServiceTestConfig) { + if (config instanceof LocalModulesServiceTestConfig localModulesServiceTestConfig) { if (configuredModulesServiceTestConfig != null) { throw new IllegalArgumentException( "Multiple LocalModulesServiceTestConfig instances provided"); } - configuredModulesServiceTestConfig = (LocalModulesServiceTestConfig) config; + configuredModulesServiceTestConfig = localModulesServiceTestConfig; } else { builder.add(config); } @@ -201,8 +196,10 @@ public LocalServiceTestHelper setEnvVersionId(String envVersionId) { */ public LocalServiceTestHelper setEnvInstance(String envInstance) { int intValue = Integer.parseInt(envInstance); - Preconditions.checkArgument(intValue >= LocalModulesServiceTestConfig.MAIN_INSTANCE, - "envInstanceId must be >= -1 and envInstanceId=" + envInstance); + Preconditions.checkArgument( + intValue >= LocalModulesServiceTestConfig.MAIN_INSTANCE, + "envInstanceId must be >= -1 and envInstanceId=%s", + envInstance); this.envInstance = intValue; return this; } diff --git a/api_dev/src/main/java/com/google/appengine/tools/info/AppengineSdk.java b/api_dev/src/main/java/com/google/appengine/tools/info/AppengineSdk.java index fd5b10b12..08f1f30de 100644 --- a/api_dev/src/main/java/com/google/appengine/tools/info/AppengineSdk.java +++ b/api_dev/src/main/java/com/google/appengine/tools/info/AppengineSdk.java @@ -47,12 +47,7 @@ public abstract class AppengineSdk { static boolean isDevAppServerTest; private static final FileFilter NO_HIDDEN_FILES = - new FileFilter() { - @Override - public boolean accept(File file) { - return !file.isHidden(); - } - }; + file -> !file.isHidden(); AppengineSdk() { sdkRoot = findSdkRoot(); diff --git a/api_dev/src/main/java/com/google/appengine/tools/info/Jetty121EE8Sdk.java b/api_dev/src/main/java/com/google/appengine/tools/info/Jetty121EE8Sdk.java index 393448b40..a9b7a26a9 100644 --- a/api_dev/src/main/java/com/google/appengine/tools/info/Jetty121EE8Sdk.java +++ b/api_dev/src/main/java/com/google/appengine/tools/info/Jetty121EE8Sdk.java @@ -233,7 +233,7 @@ List getJetty121SharedLibFiles() { sharedLibs.add(new File(sdkRoot, "lib/shared/jetty12/appengine-local-runtime-shared.jar")); File jettyHomeLib = new File(sdkRoot, JETTY121_HOME_LIB_PATH); - sharedLibs.add(new File(jettyHomeLib, "jetty-servlet-api-4.0.6.jar")); // this is javax.servlet + sharedLibs.add(new File(jettyHomeLib, "jetty-servlet-api-4.0.9.jar")); // this is javax.servlet // We want to match this file: "jetty-util-9.3.8.v20160314.jar" // but without hardcoding the Jetty version which is changing from time to time. diff --git a/api_dev/src/test/java/com/google/appengine/api/datastore/DatastoreApiHelperTest.java b/api_dev/src/test/java/com/google/appengine/api/datastore/DatastoreApiHelperTest.java index c944ab3c8..58a5572eb 100644 --- a/api_dev/src/test/java/com/google/appengine/api/datastore/DatastoreApiHelperTest.java +++ b/api_dev/src/test/java/com/google/appengine/api/datastore/DatastoreApiHelperTest.java @@ -140,6 +140,11 @@ public void testInternalError() throws InterruptedException, ExecutionException assertMakeAsyncCallThrows(ErrorCode.INTERNAL_ERROR, DatastoreFailureException.class); } + @Test + public void testPermissionDenied() throws InterruptedException, ExecutionException { + assertMakeAsyncCallThrows(ErrorCode.PERMISSION_DENIED, DatastoreFailureException.class); + } + @Test public void testV1Exceptions() throws Exception { assertV1Exception(Code.INVALID_ARGUMENT, "", IllegalArgumentException.class); diff --git a/api_dev/src/test/java/com/google/appengine/api/mail/MailServiceImplTest.java b/api_dev/src/test/java/com/google/appengine/api/mail/MailServiceImplTest.java index fe6215c03..151dfa4e2 100644 --- a/api_dev/src/test/java/com/google/appengine/api/mail/MailServiceImplTest.java +++ b/api_dev/src/test/java/com/google/appengine/api/mail/MailServiceImplTest.java @@ -347,9 +347,13 @@ public void testSendWithExceptions() throws Exception { assertThat(iae).hasMessageThat().contains("detail"); } - MailService.Message msg = setupSendCallWithApplicationException(ErrorCode.INTERNAL_ERROR); - IOException ioe = assertThrows(IOException.class, () -> service.send(msg)); - assertThat(ioe).hasMessageThat().isEqualTo("detail"); + MailService.Message msg1 = setupSendCallWithApplicationException(ErrorCode.INTERNAL_ERROR); + IOException ioe1 = assertThrows(IOException.class, () -> service.send(msg1)); + assertThat(ioe1).hasMessageThat().isEqualTo("detail"); + + MailService.Message msg2 = setupSendCallWithApplicationException(ErrorCode.INVALID_CONTENT_ID); + IOException ioe2 = assertThrows(IOException.class, () -> service.send(msg2)); + assertThat(ioe2).hasMessageThat().isEqualTo("detail"); } /** Tests that a message sent to admins works correctly. */ diff --git a/api_dev/src/test/java/com/google/appengine/api/taskqueue/dev/LocalTaskQueueTest.java b/api_dev/src/test/java/com/google/appengine/api/taskqueue/dev/LocalTaskQueueTest.java index 1a47bcb11..cf179a642 100644 --- a/api_dev/src/test/java/com/google/appengine/api/taskqueue/dev/LocalTaskQueueTest.java +++ b/api_dev/src/test/java/com/google/appengine/api/taskqueue/dev/LocalTaskQueueTest.java @@ -601,13 +601,7 @@ public void testEtaTooFarInTheFuture() { final long now = 100; long nowUsec = now * 1000; localService = LocalTaskQueueTestConfig.getLocalTaskQueue(); - Clock clock = - new Clock() { - @Override - public long getCurrentTime() { - return now; - } - }; + Clock clock = () -> now; initLocalTaskQueue(clock); localService.start(); diff --git a/api_dev/src/test/java/com/google/apphosting/utils/servlet/DeferredTaskServletTest.java b/api_dev/src/test/java/com/google/apphosting/utils/servlet/DeferredTaskServletTest.java index 0ad402430..15a3bb574 100644 --- a/api_dev/src/test/java/com/google/apphosting/utils/servlet/DeferredTaskServletTest.java +++ b/api_dev/src/test/java/com/google/apphosting/utils/servlet/DeferredTaskServletTest.java @@ -74,51 +74,39 @@ public void log(String msg) {} private static void initializeTasks(DeferredTaskServletTest testCase) { testCase.deferredSuccess = - new DeferredTask() { - @Override - public void run() { - ++getState().runCount; - assertEquals(getState().req, DeferredTaskContext.getCurrentRequest()); - assertEquals(getState().resp, DeferredTaskContext.getCurrentResponse()); - assertEquals(getState().servlet, DeferredTaskContext.getCurrentServlet()); - } + () -> { + ++getState().runCount; + assertEquals(getState().req, DeferredTaskContext.getCurrentRequest()); + assertEquals(getState().resp, DeferredTaskContext.getCurrentResponse()); + assertEquals(getState().servlet, DeferredTaskContext.getCurrentServlet()); }; testCase.deferredFail = - new DeferredTask() { - @Override - public void run() { - ++getState().runCount; - assertEquals(getState().req, DeferredTaskContext.getCurrentRequest()); - assertEquals(getState().resp, DeferredTaskContext.getCurrentResponse()); - assertEquals(getState().servlet, DeferredTaskContext.getCurrentServlet()); - throw new RuntimeException(); - } + () -> { + ++getState().runCount; + assertEquals(getState().req, DeferredTaskContext.getCurrentRequest()); + assertEquals(getState().resp, DeferredTaskContext.getCurrentResponse()); + assertEquals(getState().servlet, DeferredTaskContext.getCurrentServlet()); + throw new RuntimeException(); }; testCase.deferredMarkForRetry = - new DeferredTask() { - @Override - public void run() { - ++getState().runCount; - assertEquals(getState().req, DeferredTaskContext.getCurrentRequest()); - assertEquals(getState().resp, DeferredTaskContext.getCurrentResponse()); - assertEquals(getState().servlet, DeferredTaskContext.getCurrentServlet()); - DeferredTaskContext.markForRetry(); - } + () -> { + ++getState().runCount; + assertEquals(getState().req, DeferredTaskContext.getCurrentRequest()); + assertEquals(getState().resp, DeferredTaskContext.getCurrentResponse()); + assertEquals(getState().servlet, DeferredTaskContext.getCurrentServlet()); + DeferredTaskContext.markForRetry(); }; testCase.deferredFailDoNotRetry = - new DeferredTask() { - @Override - public void run() { - ++getState().runCount; - DeferredTaskContext.setDoNotRetry(true); - assertEquals(getState().req, DeferredTaskContext.getCurrentRequest()); - assertEquals(getState().resp, DeferredTaskContext.getCurrentResponse()); - assertEquals(getState().servlet, DeferredTaskContext.getCurrentServlet()); - throw new RuntimeException(); - } + () -> { + ++getState().runCount; + DeferredTaskContext.setDoNotRetry(true); + assertEquals(getState().req, DeferredTaskContext.getCurrentRequest()); + assertEquals(getState().resp, DeferredTaskContext.getCurrentResponse()); + assertEquals(getState().servlet, DeferredTaskContext.getCurrentServlet()); + throw new RuntimeException(); }; } diff --git a/api_dev/src/test/java/com/google/apphosting/utils/servlet/ParseBlobUploadFilterTest.java b/api_dev/src/test/java/com/google/apphosting/utils/servlet/ParseBlobUploadFilterTest.java index 4b00ca68f..a404f3864 100644 --- a/api_dev/src/test/java/com/google/apphosting/utils/servlet/ParseBlobUploadFilterTest.java +++ b/api_dev/src/test/java/com/google/apphosting/utils/servlet/ParseBlobUploadFilterTest.java @@ -27,9 +27,6 @@ import java.util.GregorianCalendar; import java.util.List; import java.util.Map; -import javax.servlet.FilterChain; -import javax.servlet.ServletRequest; -import javax.servlet.ServletResponse; import javax.servlet.http.HttpServletRequest; import junit.framework.TestCase; @@ -87,21 +84,18 @@ public void testParse() throws Exception { .doFilter( req, resp, - new FilterChain() { - @Override - public void doFilter(ServletRequest request, ServletResponse response) { - assertEquals("Example string.", request.getParameter("string")); - assertEquals(null, request.getParameter("image")); - - BlobstoreService blobstoreService = BlobstoreServiceFactory.getBlobstoreService(); - Map> blobs = - blobstoreService.getUploads((HttpServletRequest) request); - assertEquals(1, blobs.size()); - List keys = blobs.get("upload-0"); - assertEquals(2, keys.size()); - assertEquals(new BlobKey("blob-0"), keys.get(0)); - assertEquals(new BlobKey("blob-1"), keys.get(1)); - } + (request, unusedResponse) -> { + assertEquals("Example string.", request.getParameter("string")); + assertEquals(null, request.getParameter("image")); + + BlobstoreService blobstoreService = BlobstoreServiceFactory.getBlobstoreService(); + Map> blobs = + blobstoreService.getUploads((HttpServletRequest) request); + assertEquals(1, blobs.size()); + List keys = blobs.get("upload-0"); + assertEquals(2, keys.size()); + assertEquals(new BlobKey("blob-0"), keys.get(0)); + assertEquals(new BlobKey("blob-1"), keys.get(1)); }); } @@ -117,44 +111,41 @@ public void testParseBlobInfos() throws Exception { .doFilter( req, resp, - new FilterChain() { - @Override - public void doFilter(ServletRequest request, ServletResponse response) { - assertEquals("Example string.", request.getParameter("string")); - assertEquals(null, request.getParameter("image")); - - BlobstoreService blobstoreService = BlobstoreServiceFactory.getBlobstoreService(); - Map> blobs = - blobstoreService.getBlobInfos((HttpServletRequest) request); - assertEquals(1, blobs.size()); - List infos = blobs.get("upload-0"); - assertEquals(2, infos.size()); - - @SuppressWarnings("JavaUtilDate") - Date expectedCreationDate = - new Date( - new GregorianCalendar(2008, 11 - 1, 12, 10, 40, 00).getTimeInMillis() + 20); - BlobInfo expected1 = - new BlobInfo( - new BlobKey("blob-0"), - "image/jpeg", - expectedCreationDate, - "file-0.jpg", - 5, - "md5-hash", - "/bucket_name/some_random_filename1"); - BlobInfo expected2 = - new BlobInfo( - new BlobKey("blob-1"), - "image/jpeg", - expectedCreationDate, - "file-1.jpg", - 5, - "md5-hash"); - - assertEquals(expected1, infos.get(0)); - assertEquals(expected2, infos.get(1)); - } + (request, unusedResponse) -> { + assertEquals("Example string.", request.getParameter("string")); + assertEquals(null, request.getParameter("image")); + + BlobstoreService blobstoreService = BlobstoreServiceFactory.getBlobstoreService(); + Map> blobs = + blobstoreService.getBlobInfos((HttpServletRequest) request); + assertEquals(1, blobs.size()); + List infos = blobs.get("upload-0"); + assertEquals(2, infos.size()); + + @SuppressWarnings("JavaUtilDate") + Date expectedCreationDate = + new Date( + new GregorianCalendar(2008, 11 - 1, 12, 10, 40, 00).getTimeInMillis() + 20); + BlobInfo expected1 = + new BlobInfo( + new BlobKey("blob-0"), + "image/jpeg", + expectedCreationDate, + "file-0.jpg", + 5, + "md5-hash", + "/bucket_name/some_random_filename1"); + BlobInfo expected2 = + new BlobInfo( + new BlobKey("blob-1"), + "image/jpeg", + expectedCreationDate, + "file-1.jpg", + 5, + "md5-hash"); + + assertEquals(expected1, infos.get(0)); + assertEquals(expected2, infos.get(1)); }); } @@ -170,38 +161,35 @@ public void testParseFileInfos() throws Exception { .doFilter( req, resp, - new FilterChain() { - @Override - public void doFilter(ServletRequest request, ServletResponse response) { - assertEquals("Example string.", request.getParameter("string")); - assertEquals(null, request.getParameter("image")); - - BlobstoreService blobstoreService = BlobstoreServiceFactory.getBlobstoreService(); - Map> files = - blobstoreService.getFileInfos((HttpServletRequest) request); - assertEquals(1, files.size()); - List infos = files.get("upload-0"); - assertEquals(2, infos.size()); - - @SuppressWarnings("JavaUtilDate") - Date expectedCreationDate = - new Date( - new GregorianCalendar(2008, 11 - 1, 12, 10, 40, 00).getTimeInMillis() + 20); - FileInfo expected1 = - new FileInfo( - "image/jpeg", - expectedCreationDate, - "file-0.jpg", - 5, - "md5-hash", - "/bucket_name/some_random_filename1"); - FileInfo expected2 = - new FileInfo( - "image/jpeg", expectedCreationDate, "file-1.jpg", 5, "md5-hash", null); - - assertEquals(expected1, infos.get(0)); - assertEquals(expected2, infos.get(1)); - } + (request, unusedResponse) -> { + assertEquals("Example string.", request.getParameter("string")); + assertEquals(null, request.getParameter("image")); + + BlobstoreService blobstoreService = BlobstoreServiceFactory.getBlobstoreService(); + Map> files = + blobstoreService.getFileInfos((HttpServletRequest) request); + assertEquals(1, files.size()); + List infos = files.get("upload-0"); + assertEquals(2, infos.size()); + + @SuppressWarnings("JavaUtilDate") + Date expectedCreationDate = + new Date( + new GregorianCalendar(2008, 11 - 1, 12, 10, 40, 00).getTimeInMillis() + 20); + FileInfo expected1 = + new FileInfo( + "image/jpeg", + expectedCreationDate, + "file-0.jpg", + 5, + "md5-hash", + "/bucket_name/some_random_filename1"); + FileInfo expected2 = + new FileInfo( + "image/jpeg", expectedCreationDate, "file-1.jpg", 5, "md5-hash", null); + + assertEquals(expected1, infos.get(0)); + assertEquals(expected2, infos.get(1)); }); } } diff --git a/appengine_init/src/main/java/com/google/appengine/init/AppEngineWebXmlInitialParse.java b/appengine_init/src/main/java/com/google/appengine/init/AppEngineWebXmlInitialParse.java index 86b720bdd..cb6b68b84 100644 --- a/appengine_init/src/main/java/com/google/appengine/init/AppEngineWebXmlInitialParse.java +++ b/appengine_init/src/main/java/com/google/appengine/init/AppEngineWebXmlInitialParse.java @@ -20,6 +20,7 @@ import java.io.InputStream; import java.util.Objects; import java.util.Properties; +import java.util.function.UnaryOperator; import java.util.logging.Level; import java.util.logging.Logger; import javax.xml.namespace.QName; @@ -34,6 +35,10 @@ public final class AppEngineWebXmlInitialParse { private static final Logger logger = Logger.getLogger(AppEngineWebXmlInitialParse.class.getName()); + + /** Provider for environment variables, allowing for substitution in tests. */ + private UnaryOperator envProvider = System::getenv; + private String runtimeId = ""; private final String appEngineWebXmlFile; @@ -128,7 +133,7 @@ public final class AppEngineWebXmlInitialParse { public void handleRuntimeProperties() { // See if the Mendel experiment to enable HttpConnector is set automatically via env var: - if (Objects.equals(System.getenv("EXPERIMENT_ENABLE_HTTP_CONNECTOR_FOR_JAVA"), "true")) { + if (Objects.equals(envProvider.apply("EXPERIMENT_ENABLE_HTTP_CONNECTOR_FOR_JAVA"), "true")) { System.setProperty("appengine.ignore.cancelerror", "true"); System.setProperty("appengine.use.HttpConnector", "true"); } @@ -144,7 +149,7 @@ public void handleRuntimeProperties() { && event.asStartElement().getName().getLocalPart().equals(RUNTIME)) { XMLEvent runtime = reader.nextEvent(); if (runtime.isCharacters()) { - runtimeId = runtime.asCharacters().getData(); + runtimeId = runtime.asCharacters().getData().trim(); appEngineWebXmlProperties.setProperty("GAE_RUNTIME", runtimeId); } } @@ -170,7 +175,10 @@ public void handleRuntimeProperties() { } } // reset runtimeId to the value possibly overridden by system properties - runtimeId = systemProps.getProperty("GAE_RUNTIME"); + runtimeId = System.getProperty("GAE_RUNTIME"); + if (runtimeId != null) { + runtimeId = runtimeId.trim(); + } if ((Objects.equals(runtimeId, "java17") || Objects.equals(runtimeId, "java21")) && Boolean.parseBoolean(System.getProperty("appengine.use.EE10", "false")) @@ -208,7 +216,7 @@ public void handleRuntimeProperties() { System.setProperty( "appengine.use.EE8", String.valueOf( - Objects.equals(System.getenv("EXPERIMENT_ENABLE_JETTY12_FOR_JAVA"), "true"))); + Objects.equals(envProvider.apply("EXPERIMENT_ENABLE_JETTY12_FOR_JAVA"), "true"))); } else if (Objects.equals(runtimeId, "java21")) { if (Boolean.parseBoolean(System.getProperty("appengine.use.jetty121", "false"))) { System.setProperty("appengine.use.EE11", "true"); @@ -233,6 +241,33 @@ public void handleRuntimeProperties() { if (Objects.equals(runtimeId, "java25") && Boolean.getBoolean("appengine.use.EE10")) { throw new IllegalArgumentException("appengine.use.EE10 is not supported in Jetty121"); } + + // Log the runtime configuration so we can see it in the app logs. + StringBuilder configLog = + new StringBuilder("AppEngine runtime configuration: runtimeId=").append(runtimeId); + if (Objects.equals(envProvider.apply("EXPERIMENT_ENABLE_JETTY12_FOR_JAVA"), "true")) { + configLog.append(", with Jetty 12"); + } + if (Objects.equals(envProvider.apply("EXPERIMENT_ENABLE_HTTP_CONNECTOR_FOR_JAVA"), "true")) { + configLog.append(", with HTTP Connector"); + } + int initialLength = configLog.length(); + if (Boolean.getBoolean("appengine.use.EE8")) { + configLog.append(", appengine.use.EE8=true"); + } + if (Boolean.getBoolean("appengine.use.EE10")) { + configLog.append(", appengine.use.EE10=true"); + } + if (Boolean.getBoolean("appengine.use.EE11")) { + configLog.append(", appengine.use.EE11=true"); + } + if (Boolean.getBoolean("appengine.use.jetty121")) { + configLog.append(", appengine.use.jetty121=true"); + } + if (configLog.length() == initialLength) { + configLog.append(" no extra flag set"); + } + logger.info(configLog.toString()); } /** @@ -283,4 +318,8 @@ public AppEngineWebXmlInitialParse(String file) { new Object[] {BUILD_TIMESTAMP, GIT_HASH, BUILD_VERSION}); } } + + void setEnvProvider(UnaryOperator envProvider) { + this.envProvider = envProvider; + } } diff --git a/appengine_init/src/test/java/com/google/appengine/init/AppEngineWebXmlInitialParseTest.java b/appengine_init/src/test/java/com/google/appengine/init/AppEngineWebXmlInitialParseTest.java index 39c4d917c..d8a1c45b4 100644 --- a/appengine_init/src/test/java/com/google/appengine/init/AppEngineWebXmlInitialParseTest.java +++ b/appengine_init/src/test/java/com/google/appengine/init/AppEngineWebXmlInitialParseTest.java @@ -19,12 +19,19 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import java.io.IOException; import java.io.Writer; import java.nio.file.Files; import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.logging.Handler; +import java.util.logging.LogRecord; +import java.util.logging.Logger; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -99,6 +106,43 @@ public void testJava17() throws IOException { assertFalse(Boolean.getBoolean("appengine.use.jetty121")); } + @Test + public void testJava17WithJetty121() throws IOException { + createTempAppEngineWebXml( + """ + + java17 + + + + + """); + new AppEngineWebXmlInitialParse(tempFile.toFile().getAbsolutePath()).handleRuntimeProperties(); + assertFalse(Boolean.getBoolean("appengine.use.EE8")); + assertFalse(Boolean.getBoolean("appengine.use.EE10")); + assertFalse(Boolean.getBoolean("appengine.use.EE11")); + assertTrue(Boolean.getBoolean("appengine.use.jetty121")); + } + + @Test + public void testJava17WithEE8AndJetty121() throws IOException { + createTempAppEngineWebXml( + """ + + java17 + + + + + + """); + new AppEngineWebXmlInitialParse(tempFile.toFile().getAbsolutePath()).handleRuntimeProperties(); + assertTrue(Boolean.getBoolean("appengine.use.EE8")); + assertFalse(Boolean.getBoolean("appengine.use.EE10")); + assertFalse(Boolean.getBoolean("appengine.use.EE11")); + assertTrue(Boolean.getBoolean("appengine.use.jetty121")); + } + @Test public void testJava17EE10() throws IOException { createTempAppEngineWebXml( @@ -137,6 +181,43 @@ public void testJava21() throws IOException { assertFalse(Boolean.getBoolean("appengine.use.jetty121")); } + @Test + public void testJava21WithEE10Explicit() throws IOException { + createTempAppEngineWebXml( + """ + + java21 + + + + + """); + new AppEngineWebXmlInitialParse(tempFile.toFile().getAbsolutePath()).handleRuntimeProperties(); + assertFalse(Boolean.getBoolean("appengine.use.EE8")); + assertTrue(Boolean.getBoolean("appengine.use.EE10")); + assertFalse(Boolean.getBoolean("appengine.use.EE11")); + assertFalse(Boolean.getBoolean("appengine.use.jetty121")); + } + + @Test + public void testJava21WithEE8AndJetty121() throws IOException { + createTempAppEngineWebXml( + """ + + java21 + + + + + + """); + new AppEngineWebXmlInitialParse(tempFile.toFile().getAbsolutePath()).handleRuntimeProperties(); + assertTrue(Boolean.getBoolean("appengine.use.EE8")); + assertFalse(Boolean.getBoolean("appengine.use.EE10")); + assertFalse(Boolean.getBoolean("appengine.use.EE11")); + assertTrue(Boolean.getBoolean("appengine.use.jetty121")); + } + @Test public void testJava21EE8() throws IOException { createTempAppEngineWebXml( @@ -326,4 +407,436 @@ public void testJava21WithEE11() throws IOException { assertFalse(Boolean.getBoolean("appengine.use.EE10")); assertTrue(Boolean.getBoolean("appengine.use.jetty121")); } + + @Test + public void testJava17WithExperimentEnableJetty12() throws IOException { + createTempAppEngineWebXml( + """ + + java17 + + """); + AppEngineWebXmlInitialParse parser = + new AppEngineWebXmlInitialParse(tempFile.toFile().getAbsolutePath()); + parser.setEnvProvider( + key -> { + if (Objects.equals(key, "EXPERIMENT_ENABLE_JETTY12_FOR_JAVA")) { + return "true"; + } + return null; + }); + parser.handleRuntimeProperties(); + assertTrue(Boolean.getBoolean("appengine.use.EE8")); + assertFalse(Boolean.getBoolean("appengine.use.EE10")); + assertFalse(Boolean.getBoolean("appengine.use.EE11")); + assertFalse(Boolean.getBoolean("appengine.use.jetty121")); + } + + @Test + public void testJava17WithWhitespace() throws IOException { + createTempAppEngineWebXml( + """ + + + java17 + + + """); + AppEngineWebXmlInitialParse parser = + new AppEngineWebXmlInitialParse(tempFile.toFile().getAbsolutePath()); + parser.setEnvProvider( + key -> { + if (Objects.equals(key, "EXPERIMENT_ENABLE_JETTY12_FOR_JAVA")) { + return "true"; + } + return null; + }); + parser.handleRuntimeProperties(); + assertTrue(Boolean.getBoolean("appengine.use.EE8")); + } + + @Test + public void testHttpConnectorExperiment() throws IOException { + createTempAppEngineWebXml( + """ + + java17 + + """); + AppEngineWebXmlInitialParse parser = + new AppEngineWebXmlInitialParse(tempFile.toFile().getAbsolutePath()); + parser.setEnvProvider( + key -> { + if (Objects.equals(key, "EXPERIMENT_ENABLE_HTTP_CONNECTOR_FOR_JAVA")) { + return "true"; + } + return null; + }); + parser.handleRuntimeProperties(); + assertTrue(Boolean.getBoolean("appengine.use.HttpConnector")); + assertTrue(Boolean.getBoolean("appengine.ignore.cancelerror")); + } + + @Test + public void testMultipleEEFlags() throws IOException { + createTempAppEngineWebXml( + """ + + java17 + + + + + + """); + assertThrows( + IllegalArgumentException.class, + () -> + new AppEngineWebXmlInitialParse(tempFile.toFile().getAbsolutePath()) + .handleRuntimeProperties()); + } + + @Test + public void testJava25WithEE10() throws IOException { + createTempAppEngineWebXml( + """ + + java25 + + + + + """); + assertThrows( + IllegalArgumentException.class, + () -> + new AppEngineWebXmlInitialParse(tempFile.toFile().getAbsolutePath()) + .handleRuntimeProperties()); + } + + @Test + public void testJava17EE11() throws IOException { + createTempAppEngineWebXml( + """ + + java17 + + + + + """); + new AppEngineWebXmlInitialParse(tempFile.toFile().getAbsolutePath()).handleRuntimeProperties(); + assertTrue(Boolean.getBoolean("appengine.use.EE11")); + assertFalse(Boolean.getBoolean("appengine.use.EE8")); + assertFalse(Boolean.getBoolean("appengine.use.EE10")); + assertTrue(Boolean.getBoolean("appengine.use.jetty121")); + } + + @Test + public void testJava17EE8() throws IOException { + createTempAppEngineWebXml( + """ + + java17 + + + + + """); + new AppEngineWebXmlInitialParse(tempFile.toFile().getAbsolutePath()).handleRuntimeProperties(); + assertTrue(Boolean.getBoolean("appengine.use.EE8")); + assertFalse(Boolean.getBoolean("appengine.use.EE10")); + assertFalse(Boolean.getBoolean("appengine.use.EE11")); + assertFalse(Boolean.getBoolean("appengine.use.jetty121")); + } + + @Test + public void testJava25EE11() throws IOException { + createTempAppEngineWebXml( + """ + + java25 + + + + + """); + new AppEngineWebXmlInitialParse(tempFile.toFile().getAbsolutePath()).handleRuntimeProperties(); + assertTrue(Boolean.getBoolean("appengine.use.EE11")); + assertFalse(Boolean.getBoolean("appengine.use.EE8")); + assertFalse(Boolean.getBoolean("appengine.use.EE10")); + assertTrue(Boolean.getBoolean("appengine.use.jetty121")); + } + + @Test + public void testSystemPropertyOverrideRuntime() throws IOException { + createTempAppEngineWebXml( + """ + + java17 + + """); + System.setProperty("GAE_RUNTIME", "java21"); + new AppEngineWebXmlInitialParse(tempFile.toFile().getAbsolutePath()).handleRuntimeProperties(); + // Should behave like java21 (EE10=true) + assertTrue(Boolean.getBoolean("appengine.use.EE10")); + } + + @Test + public void testSystemPropertyOverrideRuntimeWithWhitespace() throws IOException { + createTempAppEngineWebXml( + """ + + java17 + + """); + System.setProperty("GAE_RUNTIME", " java21 "); + new AppEngineWebXmlInitialParse(tempFile.toFile().getAbsolutePath()).handleRuntimeProperties(); + // Should behave like java21 + assertTrue(Boolean.getBoolean("appengine.use.EE10")); + } + + @Test + public void testLogEE8() throws IOException { + Logger logger = Logger.getLogger(AppEngineWebXmlInitialParse.class.getName()); + TestHandler handler = new TestHandler(); + logger.addHandler(handler); + try { + createTempAppEngineWebXml( + """ + + java17 + + + + + """); + new AppEngineWebXmlInitialParse(tempFile.toFile().getAbsolutePath()).handleRuntimeProperties(); + boolean found = false; + for (LogRecord record : handler.records) { + if (record.getMessage().contains("appengine.use.EE8=true")) { + found = true; + break; + } + } + assertTrue("Log message should contain appengine.use.EE8=true", found); + } finally { + logger.removeHandler(handler); + } + } + + @Test + public void testLogEE10() throws IOException { + Logger logger = Logger.getLogger(AppEngineWebXmlInitialParse.class.getName()); + TestHandler handler = new TestHandler(); + logger.addHandler(handler); + try { + createTempAppEngineWebXml( + """ + + java21 + + """); + new AppEngineWebXmlInitialParse(tempFile.toFile().getAbsolutePath()).handleRuntimeProperties(); + boolean found = false; + for (LogRecord record : handler.records) { + if (record.getMessage().contains("appengine.use.EE10=true")) { + found = true; + break; + } + } + assertTrue("Log message should contain appengine.use.EE10=true", found); + } finally { + logger.removeHandler(handler); + } + } + + @Test + public void testLogEE11() throws IOException { + Logger logger = Logger.getLogger(AppEngineWebXmlInitialParse.class.getName()); + TestHandler handler = new TestHandler(); + logger.addHandler(handler); + try { + createTempAppEngineWebXml( + """ + + java25 + + """); + new AppEngineWebXmlInitialParse(tempFile.toFile().getAbsolutePath()).handleRuntimeProperties(); + boolean found = false; + for (LogRecord record : handler.records) { + if (record.getMessage().contains("appengine.use.EE11=true")) { + found = true; + break; + } + } + assertTrue("Log message should contain appengine.use.EE11=true", found); + } finally { + logger.removeHandler(handler); + } + } + + @Test + public void testLogJetty121() throws IOException { + Logger logger = Logger.getLogger(AppEngineWebXmlInitialParse.class.getName()); + TestHandler handler = new TestHandler(); + logger.addHandler(handler); + try { + createTempAppEngineWebXml( + """ + + java25 + + """); + new AppEngineWebXmlInitialParse(tempFile.toFile().getAbsolutePath()).handleRuntimeProperties(); + boolean found = false; + for (LogRecord record : handler.records) { + if (record.getMessage().contains("appengine.use.jetty121=true")) { + found = true; + break; + } + } + assertTrue("Log message should contain appengine.use.jetty121=true", found); + } finally { + logger.removeHandler(handler); + } + } + + @Test + public void testLogAllFlagsFalse() throws IOException { + Logger logger = Logger.getLogger(AppEngineWebXmlInitialParse.class.getName()); + TestHandler handler = new TestHandler(); + logger.addHandler(handler); + try { + createTempAppEngineWebXml( + """ + + java17 + + """); + new AppEngineWebXmlInitialParse(tempFile.toFile().getAbsolutePath()).handleRuntimeProperties(); + boolean found = false; + for (LogRecord record : handler.records) { + if (record.getMessage().contains(" no extra flag set")) { + found = true; + break; + } + } + assertTrue("Log message should contain ' no extra flag set'", found); + } finally { + logger.removeHandler(handler); + } + } + + @Test + public void testLogNotAllFlagsFalseWhenFlagIsSet() throws IOException { + Logger logger = Logger.getLogger(AppEngineWebXmlInitialParse.class.getName()); + TestHandler handler = new TestHandler(); + logger.addHandler(handler); + try { + createTempAppEngineWebXml( + """ + + java21 + + """); + new AppEngineWebXmlInitialParse(tempFile.toFile().getAbsolutePath()).handleRuntimeProperties(); + boolean found = false; + for (LogRecord record : handler.records) { + if (record.getMessage().contains(" no extra flag set")) { + found = true; + break; + } + } + assertFalse("Log message should NOT contain ' no extra flag set'", found); + } finally { + logger.removeHandler(handler); + } + } + + @Test + public void testLogExperimentJetty12() throws IOException { + Logger logger = Logger.getLogger(AppEngineWebXmlInitialParse.class.getName()); + TestHandler handler = new TestHandler(); + logger.addHandler(handler); + try { + createTempAppEngineWebXml( + """ + + java17 + + """); + AppEngineWebXmlInitialParse parser = + new AppEngineWebXmlInitialParse(tempFile.toFile().getAbsolutePath()); + parser.setEnvProvider( + key -> { + if (Objects.equals(key, "EXPERIMENT_ENABLE_JETTY12_FOR_JAVA")) { + return "true"; + } + return null; + }); + parser.handleRuntimeProperties(); + boolean found = false; + for (LogRecord record : handler.records) { + if (record.getMessage().contains("with Jetty 12")) { + found = true; + break; + } + } + assertTrue("Log message should contain with Jetty 12", found); + } finally { + logger.removeHandler(handler); + } + } + + @Test + public void testLogExperimentHttpConnector() throws IOException { + Logger logger = Logger.getLogger(AppEngineWebXmlInitialParse.class.getName()); + TestHandler handler = new TestHandler(); + logger.addHandler(handler); + try { + createTempAppEngineWebXml( + """ + + java17 + + """); + AppEngineWebXmlInitialParse parser = + new AppEngineWebXmlInitialParse(tempFile.toFile().getAbsolutePath()); + parser.setEnvProvider( + key -> { + if (Objects.equals(key, "EXPERIMENT_ENABLE_HTTP_CONNECTOR_FOR_JAVA")) { + return "true"; + } + return null; + }); + parser.handleRuntimeProperties(); + boolean found = false; + for (LogRecord record : handler.records) { + if (record.getMessage().contains("with HTTP Connector")) { + found = true; + break; + } + } + assertTrue("Log message should contain with HTTP Connector", found); + } finally { + logger.removeHandler(handler); + } + } + + private static class TestHandler extends Handler { + final List records = new ArrayList<>(); + + @Override + public void publish(LogRecord record) { + records.add(record); + } + + @Override + public void flush() {} + + @Override + public void close() throws SecurityException {} + } } diff --git a/appengine_testing_tests/src/test/java/com/google/appengine/tools/development/testing/LocalServiceTestHelperTest.java b/appengine_testing_tests/src/test/java/com/google/appengine/tools/development/testing/LocalServiceTestHelperTest.java index d38f808a7..5f3f42b84 100644 --- a/appengine_testing_tests/src/test/java/com/google/appengine/tools/development/testing/LocalServiceTestHelperTest.java +++ b/appengine_testing_tests/src/test/java/com/google/appengine/tools/development/testing/LocalServiceTestHelperTest.java @@ -115,13 +115,7 @@ public void testEnvProperlySet_Defaults() { @Test public void testEnvProperlySet_Custom() { - Clock alwaysEpoch = - new Clock() { - @Override - public long getCurrentTime() { - return 0; - } - }; + Clock alwaysEpoch = () -> 0; RequestMillisTimer remainingMillis = () -> 666L; LocalServiceTestHelper helper = new LocalServiceTestHelper() @@ -209,12 +203,7 @@ public void testRequestThreads() throws InterruptedException { try { Thread t = ThreadManager.createThreadForCurrentRequest( - new Runnable() { - @Override - public void run() { - threadEnv.set(ApiProxy.getCurrentEnvironment()); - } - }); + () -> threadEnv.set(ApiProxy.getCurrentEnvironment())); t.start(); t.join(JOIN_WAIT); assertFalse(t.isAlive()); @@ -233,16 +222,13 @@ public void testRequestThreadsInterruptedAtEndOfTest() { try { t = ThreadManager.createThreadForCurrentRequest( - new Runnable() { - @Override - public void run() { - while (true) { - try { - Thread.sleep(50); - } catch (InterruptedException e) { - interrupted.set(true); - break; - } + () -> { + while (true) { + try { + Thread.sleep(50); + } catch (InterruptedException e) { + interrupted.set(true); + break; } } }); diff --git a/applications/proberapp/pom.xml b/applications/proberapp/pom.xml index 88dcd244c..999194b62 100644 --- a/applications/proberapp/pom.xml +++ b/applications/proberapp/pom.xml @@ -40,7 +40,7 @@ us-central1 prober-user prober_connectivity_test_database - 2.73.1 + 2.74.0 ${project.version} target/${project.artifactId}-${project.version} @@ -59,7 +59,7 @@ com.google.cloud google-cloud-spanner - 6.107.0 + 6.108.0 com.google.appengine @@ -117,17 +117,17 @@ com.google.cloud google-cloud-bigquery - 2.57.2 + 2.58.0 com.google.cloud google-cloud-core - 2.63.1 + 2.64.0 com.google.cloud google-cloud-datastore - 2.33.2 + 2.33.3 com.google.cloud @@ -136,7 +136,7 @@ com.google.cloud google-cloud-storage - 2.62.0 + 2.62.1 com.google.cloud.sql @@ -169,7 +169,7 @@ com.mysql mysql-connector-j - 9.5.0 + 9.6.0 org.apache.httpcomponents diff --git a/applications/proberapp_jakarta/pom.xml b/applications/proberapp_jakarta/pom.xml index d42cdec88..41076f74e 100644 --- a/applications/proberapp_jakarta/pom.xml +++ b/applications/proberapp_jakarta/pom.xml @@ -40,7 +40,7 @@ us-central1 prober-user prober_connectivity_test_database - 2.73.1 + 2.74.0 ${project.version} UTF-8 target/${project.artifactId}-${project.version} @@ -60,7 +60,7 @@ com.google.cloud google-cloud-spanner - 6.107.0 + 6.108.0 com.google.appengine @@ -118,27 +118,27 @@ com.google.auth google-auth-library-oauth2-http - 1.41.0 + 1.42.1 com.google.auth google-auth-library-credentials - 1.41.0 + 1.42.1 com.google.cloud google-cloud-bigquery - 2.57.2 + 2.58.0 com.google.cloud google-cloud-core - 2.63.1 + 2.64.0 com.google.cloud google-cloud-datastore - 2.33.2 + 2.33.3 com.google.cloud @@ -147,7 +147,7 @@ com.google.cloud google-cloud-storage - 2.62.0 + 2.62.1 com.google.cloud.sql @@ -182,7 +182,7 @@ com.mysql mysql-connector-j - 9.5.0 + 9.6.0 org.apache.httpcomponents diff --git a/e2etests/testlocalapps/allinone/src/main/java/allinone/MainServlet.java b/e2etests/testlocalapps/allinone/src/main/java/allinone/MainServlet.java index 53c756469..61817e15a 100644 --- a/e2etests/testlocalapps/allinone/src/main/java/allinone/MainServlet.java +++ b/e2etests/testlocalapps/allinone/src/main/java/allinone/MainServlet.java @@ -431,12 +431,7 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws S */ private void performMathMs(int ms, PrintWriter w) { emitf(w, "Burning cpu for %d ms", ms); - runRepeatedly(ms, new Runnable() { - @Override - public void run() { - performMath(random.nextBoolean()); - } - }); + runRepeatedly(ms, () -> performMath(random.nextBoolean())); logger.info("Cpu burned"); } diff --git a/e2etests/testlocalapps/allinone_jakarta/src/main/java/allinone/MainServlet.java b/e2etests/testlocalapps/allinone_jakarta/src/main/java/allinone/MainServlet.java index edf0fb8ed..a70b9d8bd 100644 --- a/e2etests/testlocalapps/allinone_jakarta/src/main/java/allinone/MainServlet.java +++ b/e2etests/testlocalapps/allinone_jakarta/src/main/java/allinone/MainServlet.java @@ -434,14 +434,7 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp) */ private void performMathMs(int ms, PrintWriter w) { emitf(w, "Burning cpu for %d ms", ms); - runRepeatedly( - ms, - new Runnable() { - @Override - public void run() { - performMath(random.nextBoolean()); - } - }); + runRepeatedly(ms, () -> performMath(random.nextBoolean())); logger.info("Cpu burned"); } diff --git a/e2etests/testlocalapps/bundle_standard/src/main/java/servletthree/JakartaServlet3Test.java b/e2etests/testlocalapps/bundle_standard/src/main/java/servletthree/JakartaServlet3Test.java index d17a13304..0c34b880b 100644 --- a/e2etests/testlocalapps/bundle_standard/src/main/java/servletthree/JakartaServlet3Test.java +++ b/e2etests/testlocalapps/bundle_standard/src/main/java/servletthree/JakartaServlet3Test.java @@ -47,11 +47,6 @@ protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IO String prefix = getInitParameter("prefix"); String suffix = getInitParameter("suffix"); writer.println(prefix + req.getRequestURI() + suffix); - // Check we are not running with a security manager: - SecurityManager security = System.getSecurityManager(); - if (security != null) { - throw new RuntimeException("Security manager detected."); - } } } } diff --git a/e2etests/testlocalapps/bundle_standard/src/main/java/servletthree/Servlet3Test.java b/e2etests/testlocalapps/bundle_standard/src/main/java/servletthree/Servlet3Test.java index e7a8fb82e..596d9a1e4 100644 --- a/e2etests/testlocalapps/bundle_standard/src/main/java/servletthree/Servlet3Test.java +++ b/e2etests/testlocalapps/bundle_standard/src/main/java/servletthree/Servlet3Test.java @@ -49,10 +49,6 @@ protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IO String suffix = getInitParameter("suffix"); writer.println(prefix + req.getRequestURI() + suffix); // Check we are not running with a security manager: - SecurityManager security = System.getSecurityManager(); - if (security != null) { - throw new RuntimeException("Security manager detected."); - } } } } diff --git a/e2etests/testlocalapps/bundle_standard_with_no_jsp/src/main/java/servletthree/JakartaServlet3Test.java b/e2etests/testlocalapps/bundle_standard_with_no_jsp/src/main/java/servletthree/JakartaServlet3Test.java index d17a13304..0c34b880b 100644 --- a/e2etests/testlocalapps/bundle_standard_with_no_jsp/src/main/java/servletthree/JakartaServlet3Test.java +++ b/e2etests/testlocalapps/bundle_standard_with_no_jsp/src/main/java/servletthree/JakartaServlet3Test.java @@ -47,11 +47,6 @@ protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IO String prefix = getInitParameter("prefix"); String suffix = getInitParameter("suffix"); writer.println(prefix + req.getRequestURI() + suffix); - // Check we are not running with a security manager: - SecurityManager security = System.getSecurityManager(); - if (security != null) { - throw new RuntimeException("Security manager detected."); - } } } } diff --git a/lib/tools_api/src/main/java/com/google/appengine/tools/admin/AppYamlTranslator.java b/lib/tools_api/src/main/java/com/google/appengine/tools/admin/AppYamlTranslator.java index 486a73c55..766e1f610 100644 --- a/lib/tools_api/src/main/java/com/google/appengine/tools/admin/AppYamlTranslator.java +++ b/lib/tools_api/src/main/java/com/google/appengine/tools/admin/AppYamlTranslator.java @@ -148,36 +148,36 @@ private void appendIfNotZero(StringBuilder builder, String tag, double value) { private void translateAppEngineWebXml(StringBuilder builder) { if (appEngineWebXml.getAppId() != null) { - builder.append("application: '" + appEngineWebXml.getAppId() + "'\n"); + builder.append("application: '").append(appEngineWebXml.getAppId()).append("'\n"); } - builder.append("runtime: " + runtime + "\n"); + builder.append("runtime: ").append(runtime).append("\n"); if (appEngineWebXml.getUseVm()) { builder.append("vm: True\n"); } if (appEngineWebXml.isFlexible()) { - builder.append("env: " + appEngineWebXml.getEnv() + "\n"); + builder.append("env: ").append(appEngineWebXml.getEnv()).append("\n"); } if (appEngineWebXml.getEntrypoint() != null) { - builder.append("entrypoint: '" + appEngineWebXml.getEntrypoint() + "'\n"); + builder.append("entrypoint: '").append(appEngineWebXml.getEntrypoint()).append("'\n"); } if (appEngineWebXml.getRuntimeChannel() != null) { - builder.append("runtime_channel: " + appEngineWebXml.getRuntimeChannel() + "\n"); + builder.append("runtime_channel: ").append(appEngineWebXml.getRuntimeChannel()).append("\n"); } if (appEngineWebXml.getMajorVersionId() != null) { - builder.append("version: '" + appEngineWebXml.getMajorVersionId() + "'\n"); + builder.append("version: '").append(appEngineWebXml.getMajorVersionId()).append("'\n"); } if (appEngineWebXml.getService() != null) { - builder.append("service: '" + appEngineWebXml.getService() + "'\n"); + builder.append("service: '").append(appEngineWebXml.getService()).append("'\n"); } else if (appEngineWebXml.getModule() != null) { - builder.append("module: '" + appEngineWebXml.getModule() + "'\n"); + builder.append("module: '").append(appEngineWebXml.getModule()).append("'\n"); } if (appEngineWebXml.getInstanceClass() != null) { - builder.append("instance_class: " + appEngineWebXml.getInstanceClass() + "\n"); + builder.append("instance_class: ").append(appEngineWebXml.getInstanceClass()).append("\n"); } if (!appEngineWebXml.getAutomaticScaling().isEmpty()) { @@ -248,14 +248,14 @@ private void translateAppEngineWebXml(StringBuilder builder) { } builder.append(" custom_metrics:\n"); for (CustomMetricUtilization metric : settings.getCustomMetrics()) { - builder.append(" - metric_name: '" + metric.getMetricName() + "'\n"); - builder.append(" target_type: '" + metric.getTargetType() + "'\n"); + builder.append(" - metric_name: '").append(metric.getMetricName()).append("'\n"); + builder.append(" target_type: '").append(metric.getTargetType()).append("'\n"); appendIfNotNull(builder, " target_utilization: ", metric.getTargetUtilization()); appendIfNotNull(builder, " single_instance_assignment: ", metric.getSingleInstanceAssignment()); if (metric.getFilter() != null) { - builder.append(" filter: '" + metric.getFilter() + "'\n"); + builder.append(" filter: '").append(metric.getFilter()).append("'\n"); } } } @@ -264,13 +264,13 @@ private void translateAppEngineWebXml(StringBuilder builder) { if (!appEngineWebXml.getManualScaling().isEmpty()) { builder.append("manual_scaling:\n"); AppEngineWebXml.ManualScaling settings = appEngineWebXml.getManualScaling(); - builder.append(" instances: " + settings.getInstances() + "\n"); + builder.append(" instances: ").append(settings.getInstances()).append("\n"); } if (!appEngineWebXml.getBasicScaling().isEmpty()) { builder.append("basic_scaling:\n"); AppEngineWebXml.BasicScaling settings = appEngineWebXml.getBasicScaling(); - builder.append(" max_instances: " + settings.getMaxInstances() + "\n"); + builder.append(" max_instances: ").append(settings.getMaxInstances()).append("\n"); appendIfNotNull(builder, " idle_timeout: ", settings.getIdleTimeout()); } @@ -278,14 +278,14 @@ private void translateAppEngineWebXml(StringBuilder builder) { if (!services.isEmpty()) { builder.append("inbound_services:\n"); for (String service : services) { - builder.append("- " + service + "\n"); + builder.append("- ").append(service).append("\n"); } } Collection appEngineBundledServices = appEngineWebXml.getAppEngineBundledServices(); if (!appEngineBundledServices.isEmpty()) { builder.append("app_engine_bundled_services:\n"); for (String service : appEngineBundledServices) { - builder.append("- " + service + "\n"); + builder.append("- ").append(service).append("\n"); } } @@ -310,7 +310,7 @@ private void translateAppEngineWebXml(StringBuilder builder) { } if (appEngineWebXml.getAutoIdPolicy() != null) { - builder.append("auto_id_policy: " + appEngineWebXml.getAutoIdPolicy() + "\n"); + builder.append("auto_id_policy: ").append(appEngineWebXml.getAutoIdPolicy()).append("\n"); } else { // NOTE: The YAML parsing and validation done in the admin console must // set the value for unspecified auto_id_policy to 'legacy' in order to achieve @@ -327,14 +327,14 @@ private void translateAppEngineWebXml(StringBuilder builder) { if (appEngineWebXml.getVpcAccessConnector() != null) { VpcAccessConnector connector = appEngineWebXml.getVpcAccessConnector(); builder.append("vpc_access_connector:\n"); - builder.append(" name: " + connector.getName() + "\n"); + builder.append(" name: ").append(connector.getName()).append("\n"); if (connector.getEgressSetting().isPresent()) { - builder.append(" egress_setting: " + connector.getEgressSetting().get() + "\n"); + builder.append(" egress_setting: ").append(connector.getEgressSetting().get()).append("\n"); } } if (appEngineWebXml.getServiceAccount() != null) { - builder.append("service_account: " + appEngineWebXml.getServiceAccount() + "\n"); + builder.append("service_account: ").append(appEngineWebXml.getServiceAccount()).append("\n"); } List adminConsolePages = appEngineWebXml.getAdminConsolePages(); @@ -342,8 +342,8 @@ private void translateAppEngineWebXml(StringBuilder builder) { builder.append("admin_console:\n"); builder.append(" pages:\n"); for (AdminConsolePage page : adminConsolePages) { - builder.append(" - name: " + page.getName() + "\n"); - builder.append(" url: " + page.getUrl() + "\n"); + builder.append(" - name: ").append(page.getName()).append("\n"); + builder.append(" url: ").append(page.getUrl()).append("\n"); } } @@ -362,13 +362,13 @@ private void translateAppEngineWebXml(StringBuilder builder) { + fileName + ", out of " + staticFiles); } // error_handlers doesn't want a leading slash here. - builder.append("- file: __static__" + fileName + "\n"); + builder.append("- file: __static__").append(fileName).append("\n"); if (handler.getErrorCode() != null) { - builder.append(" error_code: " + handler.getErrorCode() + "\n"); + builder.append(" error_code: ").append(handler.getErrorCode()).append("\n"); } String mimeType = webXml.getMimeTypeForPath(handler.getFile()); if (mimeType != null) { - builder.append(" mime_type: " + mimeType + "\n"); + builder.append(" mime_type: ").append(mimeType).append("\n"); } } } @@ -383,7 +383,7 @@ private void translateAppEngineWebXml(StringBuilder builder) { ApiConfig apiConfig = appEngineWebXml.getApiConfig(); if (apiConfig != null) { builder.append("api_config:\n"); - builder.append(" url: " + apiConfig.getUrl() + "\n"); + builder.append(" url: ").append(apiConfig.getUrl()).append("\n"); builder.append(" script: unused\n"); } @@ -454,8 +454,11 @@ private void appendBetaSettings(Map betaSettings, StringBuilder if (betaSettings != null && !betaSettings.isEmpty()) { builder.append("beta_settings:\n"); for (Map.Entry setting : betaSettings.entrySet()) { - builder.append( - " " + yamlQuote(setting.getKey()) + ": " + yamlQuote(setting.getValue()) + "\n"); + builder.append(" ") + .append(yamlQuote(setting.getKey())) + .append(": ") + .append(yamlQuote(setting.getValue())) + .append("\n"); } } } @@ -516,7 +519,7 @@ private void appendNetwork(Network network, StringBuilder builder) { if (!network.getForwardedPorts().isEmpty()) { builder.append(" forwarded_ports:\n"); for (String forwardedPort : network.getForwardedPorts()) { - builder.append(" - " + forwardedPort + "\n"); + builder.append(" - ").append(forwardedPort).append("\n"); } } appendIfNotNull(builder, " name: ", network.getName()); @@ -660,8 +663,9 @@ public void translateGlob(StringBuilder builder, Glob glob) { (List) glob.getProperty(WELCOME_FILES, RESOLVER); if (welcomeFiles != null) { for (String welcomeFile : welcomeFiles) { - builder.append("- url: (" + regex + ")\n"); - builder.append(" static_files: __static__" + root + "\\1" + welcomeFile + "\n"); + builder.append("- url: (").append(regex).append(")\n"); + builder.append(" static_files: __static__").append(root).append("\\1").append(welcomeFile) + .append("\n"); builder.append(" upload: __NOT_USED__\n"); builder.append(" require_matching_file: True\n"); translateHandlerOptions(builder, glob); @@ -670,8 +674,8 @@ public void translateGlob(StringBuilder builder, Glob glob) { } else { Boolean isStatic = (Boolean) glob.getProperty(STATIC_PROPERTY, RESOLVER); if (isStatic != null && isStatic.booleanValue()) { - builder.append("- url: (" + regex + ")\n"); - builder.append(" static_files: __static__" + root + "\\1\n"); + builder.append("- url: (").append(regex).append(")\n"); + builder.append(" static_files: __static__").append(root).append("\\1\n"); builder.append(" upload: __NOT_USED__\n"); builder.append(" require_matching_file: True\n"); translateHandlerOptions(builder, glob); @@ -684,7 +688,7 @@ private void translateAdditionalStaticOptions(StringBuilder builder, Glob glob) throws AppEngineConfigException { String expiration = (String) glob.getProperty(EXPIRATION_PROPERTY, RESOLVER); if (expiration != null) { - builder.append(" expiration: " + expiration + "\n"); + builder.append(" expiration: ").append(expiration).append("\n"); } @SuppressWarnings("unchecked") @@ -787,7 +791,7 @@ public void translateGlob(StringBuilder builder, Glob glob) { Boolean isDynamic = (Boolean) glob.getProperty(DYNAMIC_PROPERTY, RESOLVER); if (isDynamic != null && isDynamic.booleanValue()) { - builder.append("- url: " + regex + "\n"); + builder.append("- url: ").append(regex).append("\n"); builder.append(" script: unused\n"); translateHandlerOptions(builder, glob); } diff --git a/pom.xml b/pom.xml index 39c28a9b8..5db01d311 100644 --- a/pom.xml +++ b/pom.xml @@ -69,8 +69,8 @@ 17 UTF-8 9.4.58.v20250814 - 12.0.31 - 12.1.5 + 12.0.32 + 12.1.6 2.0.17 4.33.4 https://oss.sonatype.org/content/repositories/google-snapshots/ @@ -641,7 +641,7 @@ commons-codec commons-codec - 1.20.0 + 1.21.0 @@ -685,7 +685,7 @@ com.google.cloud google-cloud-logging - 3.23.10 + 3.24.0 org.apache.commons @@ -783,7 +783,7 @@ org.apache.maven.plugins maven-compiler-plugin - 3.14.1 + 3.15.0 org.apache.maven.plugins diff --git a/protobuf/api/datamodel.proto b/protobuf/api/datamodel.proto deleted file mode 100644 index 39ef4949c..000000000 --- a/protobuf/api/datamodel.proto +++ /dev/null @@ -1,185 +0,0 @@ -/* - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Copyright 2009 Google Inc. All Rights Reserved. - -syntax = "proto2"; - -package java.apphosting; - -import "entity_bytes.proto"; - -option java_package = "com.google.appengine.tools.appstats.proto2api"; -option java_outer_classname = "StatsProtos"; -option optimize_for = SPEED; - -// Represents some quick statistics on similar RPCs, grouped by their -// service name and call name. -message AggregateRpcStatsProto { - required string service_call_name = 1; - required int64 total_amount_of_calls = 3; - // field actually contains micropennies - // TODO: Rename this field once the appstats client code is out of the - // labs jar. - optional int64 total_cost_of_calls_microdollars = 4; - repeated BilledOpProto total_billed_ops = 5; -} - -// Represents a key-value pair. -message KeyValProto { - required string key = 1; - required string value = 2; -} - -// Represents a single stack frame (python) or line in a stack trace (Java). -message StackFrameProto { - required string class_or_file_name = 1; // filename in Python, class in Java - optional int32 line_number = 2; // not always available in Java - required string function_name = 3; - repeated KeyValProto variables = 4; // not available in Java -} - -// Billed operations associated with an RPC or a collection of RPCs. -message BilledOpProto { - enum BilledOp { - DATASTORE_READ = 0; - DATASTORE_WRITE = 1; - DATASTORE_SMALL = 2; - MAIL_RECIPIENT = 3; - CHANNEL_OPEN = 4; - XMPP_STANZA = 5; - - // Implementation never finished must be preserved as it's referenced by - // the labs jar. - CHANNEL_PRESENCE = 6; - } - required BilledOp op = 1; - required int32 num_ops = 2; // the number of times that op was performed -} - -// Detailed information about individual datastore RPC calls such as keys of -// entities fetched or written by the call. In addition to the entity keys, -// useful information specific to each call is recorded. E.g., for queries, -// the entity kind and cursor information is recorded; For gets, a flag -// indicating if the requested entity key is successfully retrieved is recorded. -message DatastoreCallDetailsProto { - optional string query_kind = 1; - optional .storage_onestore_v3.Reference query_ancestor = 2; - optional fixed64 query_thiscursor = 3; - optional fixed64 query_nextcursor = 4; - - // For get calls, not all requested entities are successfully retrieved. - // We record a bool per requested entity key indicating if the corresponding - // key was successfully fetched. The actual set of entities requested is - // recorded in the keys_read field below. - repeated bool get_successful_fetch = 5; - - // Optional (resource and space intensive) information about the keys of - // entities that were fetched/written in datastore get/put/query/next - // calls. Currently, entities accessed in other RPC calls is not recorded. - // For get calls, keys_read represents the set of keys requested - // from the datastore -- success status is recorded seperately. - repeated .storage_onestore_v3.Reference keys_read = 6; - repeated .storage_onestore_v3.Reference keys_written = 7; -} - -// Represents the statistics for a single RPC in a request. -message IndividualRpcStatsProto { - required string service_call_name = 1; - optional string request_data_summary = 3; - optional string response_data_summary = 4; - optional int64 api_mcycles = 5; - optional int64 api_milliseconds = 11; - required int64 start_offset_milliseconds = 6; - optional int64 duration_milliseconds = 7 [default = 0]; - optional string namespace = 8 [default = '']; - optional bool was_successful = 9 [default = true]; - - // Optional (resource and space intensive) information about the call stack - // of the rpc invocation. - repeated StackFrameProto call_stack = 10; - - // Detailed information about individual datastore RPC calls such as keys - // of entities fetched or written by the call. - optional DatastoreCallDetailsProto datastore_details = 12; - - // field actually contains micropennies - // TODO: Rename this field once the appstats client code is out of the - // labs jar. - optional int64 call_cost_microdollars = 13; - repeated BilledOpProto billed_ops = 14; -} - -// Represents statistical data for a single request. -// This protocol buffer can contain full, verbose information, or just be -// a smaller, compact summary that takes up less space. In the latter case, -// only fields with a number lower than 100 are populated. This makes it -// very easy to gather a subset of summary-only fields using the protocol -// buffer's descriptor. -message RequestStatProto { - // The wall time at the start of processing this request. - // Format is the difference, measured in milliseconds, between the current - // time and midnight, January 1, 1970 UTC. - required int64 start_timestamp_milliseconds = 1; - - // The http method, like GET or POST. - optional string http_method = 2 [default = "GET"]; - - // The http path. - optional string http_path = 3 [default = "/"]; - - // The http query string. - optional string http_query = 4; - - // The http response code. - optional int32 http_status = 5 [default = 200]; - - // The wall time it took for the request to complete. - // The end time of the request can be computed by adding this value to - // start_timestamp_milliseconds. - required int64 duration_milliseconds = 6; - - // The total amount of time spent in API calls, in megacycles. - optional int64 api_mcycles = 7; - - // The total amount of time spent in processing the rpc, minus API calls, - // in megacycles. - optional int64 processor_mcycles = 8; - - // A quick summary of all rpc calls made. - repeated AggregateRpcStatsProto rpc_stats = 9; - - // ================================================================= - // ================================================================= - - // CGI environment variables in the request. - // This will be optional (can be turned off, since it takes up additional - // memcache space), and may be completely missing in Java. - repeated KeyValProto cgi_env = 101; - - // The amount of overhead spent in collecting extra data for statistics, - // in milliseconds wall time. - optional int64 overhead_walltime_milliseconds = 102; - - // The email of the user, if a user is logged in. - optional string user_email = 103; - - // Was the user an administrator at the time of this request? - optional bool is_admin = 104; - - // Data for each individual RPC performed in this request. - repeated IndividualRpcStatsProto individual_stats = 107; -} diff --git a/protobuf/clone-controller.proto b/protobuf/clone-controller.proto deleted file mode 100644 index 6733dd993..000000000 --- a/protobuf/clone-controller.proto +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -syntax = "proto2"; - -package apphosting; - -import "clone.proto"; -import "empty-message.proto"; - -option java_package = "com.google.apphosting.base.protos"; -option java_outer_classname = "ModelClonePb"; - -message DeadlineInfo { - required string security_ticket = 1; - // There are two deadlines: soft and hard. - // Users should have a chance to clean up after the soft deadline, - // but their code should be forcibly stopped at the hard deadline. - required bool hard = 2; -} - -message PerformanceDataRequest { - optional java.apphosting.PerformanceData.Type type = 1 - [default = PERIODIC_SAMPLE]; -} - -service CloneController { - // Asks the Clone to put itself into the stopped state, by sending - // itself a SIGSTOP when it is safe to do so. The Clone will be - // Sandboxed and resume from this point. - rpc WaitForSandbox(EmptyMessage) returns (EmptyMessage) {} - - // Updates per-app settings for this clone. - rpc ApplyCloneSettings(java.apphosting.CloneSettings) returns (EmptyMessage) { - - } - - // Notifies the clone that the soft or hard deadline for an active request - // has expired. - rpc SendDeadline(DeadlineInfo) returns (EmptyMessage) {} - - // Deprecated. - rpc GetPerformanceData(PerformanceDataRequest) - returns (java.apphosting.PerformanceData) {} -} diff --git a/protobuf/clone.proto b/protobuf/clone.proto deleted file mode 100644 index b700121ab..000000000 --- a/protobuf/clone.proto +++ /dev/null @@ -1,430 +0,0 @@ -/* - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// LINT: ALLOW_GROUPS -// Copyright 2009 Google Inc. All Rights Reserved. - -syntax = "proto2"; - -package java.apphosting; - -import "common.proto"; - -option cc_enable_arenas = true; -option java_package = "com.google.apphosting.base.protos"; -option java_outer_classname = "ClonePb"; - -// Performance data for a clone. It consists of multiple entries, each tagged -// with the format the data is in. -message PerformanceData { - enum Type { - // Unknown sample type. - UNKNOWN = 0; - - // A sample recorded after the loading request for an app. - AFTER_LOADING = 1; - - // A sample taken periodically while the app is running. - PERIODIC_SAMPLE = 2; - } - - enum Format { - // A human-readable blob of text. - HUMAN_READABLE_TEXT = 1; - - // Hotspot hsperfdata. This is a (currently 32KB) data structure whose - // contents are described in - // http://cs/depot/depot2/google_vendor_src_branch/openjdk7/trunk/hotspot/src/share/vm/runtime/perfMemory.hpp. - // Typically Java tools access it via the sun.misc.Perf class. - JAVA_HOTSPOT_HSPERFDATA = 2; - } - - message Entry { - // Identifies the format of the payload. - optional Format format = 1; - optional bytes payload = 2; - } - - repeated Entry entries = 1; - optional Type type = 2 [default = PERIODIC_SAMPLE]; -} - -// Deadline settings for an api package. -// Missing values imply that you should use the default. -message ApiPackageDeadlines { - reserved 4, 5; - - required string api_package = 1; - optional double default_deadline_s = 2; - optional double max_deadline_s = 3; -} - -// Settings intended for a clone to implement. -// -// Next tag: 21 -message CloneSettings { - // Deprecated fields which should not be re-used. - reserved 1, 2, 5, 7, 8, 9, 10, 11, 12, 13, 15, 16, 17, 18, 19; - - // Max number of outstanding APIHost RPCs allowed on the Stubby channel. - optional int32 max_outstanding_api_rpcs = 3; - - // API call deadline settings for online requests. - repeated ApiPackageDeadlines api_call_deadlines = 4; - - // API call deadline settings for offline requests. - repeated ApiPackageDeadlines offline_api_call_deadlines = 6; - - // *********************************************************** - // The following fields should not be modified with lifeguard. - // *********************************************************** - - // Publicly exposed mostly-unique identifier for this clone. See clone.proto - // for more details on how this works. - optional bytes clone_key = 14; - - // Settings for starting a clone intended to create a snapshot. - optional SnapshotSettings snapshot_settings = 20; -} - -// Settings for clones to produce sandbox snapshots. -// -// Next tag: 3 -message SnapshotSettings { - // The name of the container to snapshot. - optional string snapshot_container_name = 1; - - // Uniquely identifies an operation for creating a snapshot. - optional string op_id = 2; -} - -// A runtime-specific memory multiplier. -// -// Next tag: 3 -message MemoryMultiplier { - optional string runtime_id = 1; - optional float multiplier = 2; -} - -// Settings intended for a supervisor to implement. -// -// Next tag: 75 -message SupervisorSettings { - // Deprecated fields which should not be re-used. - reserved 1, 2, 4, 10, 13, 14, 15, 17, 22, 30, 31, 40, 42, 44, 45, 46, 47; - - // Soft virtual and private memory limits for the Clone process. When either - // of these limits are exceeded, the Clone process will be killed politely - // after handling a request. - optional int32 soft_virtual_memory_limit_mb = 3; - optional int32 soft_private_memory_limit_mb = 8; - - // Medium private memory limit for the clone process. Whereas the soft limit - // is enforced at the end of requests, this medium limit will be checked - // periodically. Clones in excess of this limit will be sent a shutdown - // request. Applies only to background servers. - optional int32 medium_private_memory_limit_mb = 35; - - // Hard private memory limit for the clone process. Whereas the soft limit is - // enforced at the end of requests, this hard limit will be checked - // periodically. Clones in excess of this limit will be killed immediately. - optional int32 hard_private_memory_limit_mb = 9; - - // Hard private memory multiplier for the clone process. This is an input to - // the calculation of the hard memory limit, and it is specified per runtime. - // The hard private memory multiplier takes precedence over the default - // per-runtime multiplier. - repeated MemoryMultiplier hard_private_memory_multiplier = 74; - - // Heap size, for java processes. - // DEPRECATED in updateable runtimes (go/runtime-updates). Appserver ignores - // the value set here, but you can customize the value of --max_jvm_heap_size - // through runtime updates. - optional int32 java_clone_heap_size_mb = 29; - - // Perm-gen size, for java processes. - // DEPRECATED in updateable runtimes (go/runtime-updates). Appserver ignores - // the value set here, but you can customize the value of - // --max_jvm_perm_gen_size through runtime updates. - optional int32 java_clone_perm_gen_size_mb = 62; - - // For java processes, this determines which VM is used. OpenJDK - // has two VMs: the client VM (lightweight and fast start-up) and - // the server VM (slower to warm up but better steady-state - // performance). The primary difference is the way that hotspot - // compilation works: the client VM compiles after fewer iterations - // (1500) but spends less time optimizing the generated machine - // code, while the server VM observes the code for many more - // iterations (10000) before compiling and optimizes more - // aggressively. If this is unset, the client compiler will be used. - enum JavaVmType { - CLIENT = 0; // java -client - SERVER = 1; // java -server - } - // DEPRECATED in updateable runtimes (go/runtime-updates). Appserver ignores - // the value set here, but you can customize the value of --vm_type through - // runtime updates. - optional JavaVmType java_vm_type = 34; - - // ShouldWaitForClone will return true if pending time (i.e. the time this - // request has already sat on the pending queue up until now) is below this - // threshold. - // - // NOTE: Also used by ShouldStartLoadingRequest as a hard floor. If - // pending time is below this threshold, then it will return false. - // - // NOTE: With 1.6.2, it is *only* used by ShouldStartLoadingRequest. - optional double wait_for_active_clones_min_pending_time_s = 39; - // ShouldWaitForClone will return false if pending time (i.e. the time this - // request has already sat on the pending queue up until now) is above this - // threshold. - // - // NOTE: Also used by ShouldStartLoadingRequest as a hard ceiling. If - // pending time is above this threshold, then it will return true. - // - // NOTE: With 1.6.2, it is *only* used by ShouldStartLoadingRequest. - optional double wait_for_active_clones_max_pending_time_s = 41; - - // Max requests a Clone can serve before it is killed. - optional int32 max_clone_successful_requests = 5; - - // Max sequential errors a Clone can serve before it is killed. - optional int32 max_clone_sequential_errors = 6; - - // Maximum number of concurrent outstanding (aka active) requests to allow for - // this app or version on this appserver. - optional int32 max_active_requests = 7; - - // Maximum number of active clones to allow for this app or version on this - // appserver. An active clone is any clone that is currently processing at - // least one request. - optional int32 max_active_clones = 27; - - // Determines whether the appserver will throttle loading requests for this - // app version. Default: true for non-servers, false for servers. - optional bool throttle_loading_requests = 26; - - // Maximum number of recently started concurrent loading requests to allow. - optional int32 max_loading_requests = 19; - - // N.B.(jonmac): For the request deadline parameters below, we will end up - // using the maximum applicable deadline. For example, if a request is a - // offline-warming-loading request (yes, this is possible), then the request - // will be given the deadline which is the max of all deadline options below. - // If the request is a warm offline request, it would be given a deadline - // which is the max of request_deadline_s and offline_request_deadline_s. - - // Overrides the time a runtime is given to process a request. Excludes time - // spent in a pending queue, and only refers to time spent in the runtime. - optional double request_deadline_s = 11; - - // Same as request_deadline_s, but for offline requests. - optional double offline_request_deadline_s = 18; - - // Same as request_deadline_s, but for warming requests. - optional double warming_request_deadline_s = 20; - - // Same as request_deadline_s, but for loading requests. - optional double loading_request_deadline_s = 32; - - // Same as request_deadline_s, but for shutdown requests. - optional double shutdown_request_deadline_s = 33; - - // Overrides the time to allow a request to sit in the pending queue. - optional double max_pending_delay_s = 12; - - // Boost factor for max_pending_delay_s. Don't use if we don't have to - // b/122049200 - optional double loading_max_pending_delay_boost_factor = 73; - - // Allows the request to sit in a pending queue up to `max pending delay` - // times max_pending_delay_s_boost_limit_factor if a loading request for - // this appversion is being executed. 0 effectively disables this - // functionality for the app. - optional double max_pending_delay_boost_limit_factor = 70; - - // Maximum number of pending requests for an app version on one - // appserver. Once this is exceeded we will begin returning PUSH_BACK to new - // requests to a given app version. - optional int32 push_back_max_pending_requests = 16; - - // Maximum cpu rate to allow for any clone belonging to this version, enforced - // by the throttler. - // TODO(b/125948820) although this field is called a rate, it is treated in - // the code as a mhz value. - optional double max_cpu_rate = 21; - - // Padding for maximum cpu rate of this clone. It will be ignored if - // max_cpu_rate is set. Otherwise, it will be added to the instance class - // specific mcycle value as defined by --instance_class_cpu_mhz. - // TODO(b/125948820) although this field is called a rate, it is treated in - // the code as a mhz value. - optional double cpu_rate_padding = 71; - - // CPU rate to start any clone belonging to this version, and held until - // first loading request is finished or a timeout. - // - // Note that this value is NOT mcycles. To prevent accidental - // misconfiguration, this value is sanity-checked in version_settings.cc. - optional double startup_cpu_rate = 69; - - // Maximum number of concurrent non-background requests one clone process may - // serve. - optional int32 max_concurrent_requests = 23; - - // Number of cpu shares for the clone to have. - optional int32 cpu_shares = 68; - - // Minimum number of concurrent non-background requests one clone process may - // serve. - optional int32 min_concurrent_requests = 54; - - // Maximum number of concurrent background requests one clone process may - // serve. - optional int32 max_background_requests = 53; - - // Maximum size of all http requests one clone process may serve concurrently. - // Not enforced strictly. Rather, the clone can accept new requests if under - // this threshold, but once the clone is over this threshold it cannot accept - // additional requests. - optional int64 max_concurrent_requests_total_size = 36; - - // Limit the number of concurrent requests to a clone based on its past CPU - // usage instead of relying on a static max_concurrent_requests and - // max_accepting_cpu. - optional bool use_dynamic_max_concurrent_requests = 55; - - // The maximum fraction of a clone's CPU rate that is in use before the - // dynamic max concurrent requests is lowered. - optional double dynamic_max_concurrent_requests_max_cpu_fraction = 56; - - // The minimum fraction of a clone's CPU rate that is in use before the - // dynamic max concurrent requests is raised. - optional double dynamic_max_concurrent_requests_min_cpu_fraction = 57; - - // Maximum size of all pending http requests for an app version. Not enforced - // strictly. Rather, the pending queue can accept new requests if under this - // threshold, but once the pending queue is over this threshold, additional - // requests will fail immediately with PENDING_QUEUE_TOO_LARGE. - optional int64 max_pending_requests_total_size = 37; - - // If pending queue for app version is above this value, then push-back - // against incoming requests if possible. - optional int64 push_back_max_pending_requests_total_size = 38; - - // If positive, then if the clone is running at a cpu rate higher than this - // (over the last second), we will not send it additional requests, even if it - // has more space for another request per max_concurrent_requests. - optional double max_accepting_cpu = 25; - - // How to balance requests across clones for this app version on a given - // appserver. - // Type: apphosting::CloneBalancingPolicy::Enum. Can't use that directly - // because of http://wiki/Main/Proto2WithGenproto, the default value, and - // apphosting/release/BUILD. Boo! - // Clone-scheduled apps ignore this field, and use LEAST_ACTIVE by default - // (eaglepush-controlled). - optional int32 clone_lb_policy = 24 [default = 0]; - - // The default profiler settings to use if no per-request header is specified. - optional ProfilerSettings profiler_settings = 48; - - // Parameters that tune the ShouldStartLoadingRequest algorithm. Only active - // if use_multi_queue. - // - // ShouldStartLoadingRequest will compare (pending-time + predicted-run-time) - // against (average-warm-latency * 'multi_queue_warm_latency_multiplier'). If - // the predicted time is less, then it will wait; if greater, it will issue a - // loading request. - optional double multi_queue_warm_latency_multiplier = 50; - // Hard floor for ShouldStartLoadingRequest. If (pending-time + - // predicted-run-time) is below this threshold, it will return false. - optional double multi_queue_min_predicted_time_s = 51; - // Hard ceiling for ShouldStartLoadingRequest. If (pending-time + - // predicted-run-time) is above this threshold, it will return true. - optional double multi_queue_max_predicted_time_s = 52; - - // The sliding-scale-routing algorithm settings. - // - // This algorithm is part of the PUSH_BACK system. It is designed primarily - // for apps scheduled onto many appservers. - // - // Traditional pfe->appserver routing is RANDOM. A pfe will pick an - // appserver, and send it an AppHttpRequest. The appserver will then PUSH_BACK - // if its cpu is too high or any number of other PUSH_BACK conditions are - // triggered. The pfe would then send the request to another appserver, for up - // to a total of --max_appserver_retries retries (in prod this is set to 11). - // - // The sliding-scale-algorithm leverages this system together with - // RequestOracle to do more intelligent load balancing. - // - // When enabled, the first 'ready_clone_attempts' AppHttpRequests for a single - // user request will PUSH_BACK if there is no ready clone available for this - // app version on this appserver. Note that for clone scheduler managed clones - // the first sliding_scale_ready_non_discretionary_clone_attempts will only - // look for ready reserved clones. (ie, those requested by clone scheduler). - // This is in addition to the rest of the PUSH_BACK conditions. - // After that, the next 'low_predicted_time_attempts' - // for a single user request will PUSH_BACK if the predicted pending time for - // the request exceeds a proportional target. An example helps to illustrate. - // - // In this example, this app is on more than 11 appservers, - // --max_appserver_retries=11, ready_clone_attempts=2, - // low_predicted_time_attempts=5, and sliding_scale_max_predicted_time_s=1000. - // This results in the following flow: - // - // Attempt Number | AppServer will PUSH_BACK if... - // ---------------------------------------------------------- - // 0 | app version has no ready clones - // 1 | app version has no ready clones - // 2 | predicted pending time is >200ms - // 3 | predicted pending time is >400ms - // 4 | predicted pending time is >600ms - // 5 | predicted pending time is >800ms - // 6 | predicted pending time is >1000ms - // 7..11 | normal PUSH_BACK logic, nothing extra - // - // The goal of this routing logic is to route around hot appservers, find - // ready clones. A nice property is that in the steady state, no extra work is - // involved. It's only when there will be pending delays does the pfe perform - // extra hops for the request, in proportion to the extent of the pending - // delay. - optional bool enable_sliding_scale_routing = 58; - optional int32 sliding_scale_ready_clone_attempts = 59; - optional int32 sliding_scale_ready_non_discretionary_clone_attempts = 72; - optional int32 sliding_scale_low_predicted_time_attempts = 60; - optional double sliding_scale_max_predicted_time_s = 61; - // If present, only applies during the ready_clone phase. Will trigger - // sliding-scale PUSH_BACK if the app version is not under this active clone - // threshold. - optional int32 sliding_scale_max_active_clones = 63; - - // The clone-reducer algorithm settings. - // - // This runs independent of the existing LRU eviction and excess-idle-clone - // algorithms, and its purpose is to cull unnecessary clones from the cache. - // Always ignores reserved clones. - optional bool enable_clone_reducer = 64; - // Minimum age of clones to consider for culling. - optional int32 clone_reducer_min_age_s = 65; - // Minimum desired utilization of clones. This is a ratio threshold, and - // evaluated against the rate1m of active_time as measured in sec/sec. Clones - // with utilizations below this will be culled. - optional double clone_reducer_min_utilization_1m = 66; - // Do not cull an app version unless it has more non-reserved clones than - // this. - optional int32 clone_reducer_min_clones = 67; -} diff --git a/protobuf/source_context.proto b/protobuf/source_context.proto deleted file mode 100644 index 86c38548d..000000000 --- a/protobuf/source_context.proto +++ /dev/null @@ -1,176 +0,0 @@ -/* - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -syntax = "proto3"; - -package google.devtools.source.v1; - -option csharp_namespace = "Google.Cloud.DevTools.Source.V1"; -option php_namespace = "Google\\Cloud\\DevTools\\Source\\V1"; -option java_multiple_files = true; -option java_package = "com.google.apphosting.base.protos"; -option java_outer_classname = "SourceContextProto"; -option cc_enable_arenas = true; - -// A SourceContext is a reference to a tree of files. A SourceContext together -// with a path point to a unique revision of a single file or directory. -message SourceContext { - // A SourceContext can refer any one of the following types of repositories. - oneof context { - // A SourceContext referring to a revision in a cloud repo. - CloudRepoSourceContext cloud_repo = 1; - - // A SourceContext referring to a snapshot in a cloud workspace. - CloudWorkspaceSourceContext cloud_workspace = 2; - - // A SourceContext referring to a Gerrit project. - GerritSourceContext gerrit = 3; - - // A SourceContext referring to any third party Git repo (e.g. GitHub). - GitSourceContext git = 6; - } -} - -// An ExtendedSourceContext is a SourceContext combined with additional -// details describing the context. -message ExtendedSourceContext { - // Any source context. - SourceContext context = 1; - - // Labels with user defined metadata. - map labels = 2; -} - -// An alias to a repo revision. -message AliasContext { - // The type of an Alias. (-- This enum should be kept in sync with the - // Warehouse LabelType enum in google3/codesite/proto/browse.proto and - // the Kind enum in google3/google/devtools/source/v1/resources.proto --) - enum Kind { - ANY = 0; // Do not use. - FIXED = 1; // Git tag - MOVABLE = 2; // Git branch - // OTHER is used to specify non-standard aliases, those not of the kinds - // above. For example, if a Git repo has a ref named "refs/foo/bar", it - // is considered to be of kind OTHER. - OTHER = 4; - } - - // The alias kind. - Kind kind = 1; - - // The alias name. - string name = 2; -} - -// A CloudRepoSourceContext denotes a particular revision in a cloud -// repo (a repo hosted by the Google Cloud Platform). -message CloudRepoSourceContext { - // The ID of the repo. - RepoId repo_id = 1; - - // A revision in a cloud repository can be identified by either its revision - // ID or its Alias. - oneof revision { - // A revision ID. - string revision_id = 2; - - // The name of an alias (branch, tag, etc.). - string alias_name = 3 [deprecated = true]; - - // An alias, which may be a branch or tag. - AliasContext alias_context = 4; - } -} - -// A CloudWorkspaceSourceContext denotes a workspace at a particular snapshot. -message CloudWorkspaceSourceContext { - // The ID of the workspace. - CloudWorkspaceId workspace_id = 1; - - // The ID of the snapshot. - // An empty snapshot_id refers to the most recent snapshot. - string snapshot_id = 2; -} - -// A SourceContext referring to a Gerrit project. -message GerritSourceContext { - // The URI of a running Gerrit instance. - string host_uri = 1; - - // The full project name within the host. Projects may be nested, so - // "project/subproject" is a valid project name. - // The "repo name" is hostURI/project. - string gerrit_project = 2; - - // A revision in a Gerrit project can be identified by either its revision ID - // or its alias. - oneof revision { - // A revision (commit) ID. - string revision_id = 3; - - // The name of an alias (branch, tag, etc.). - string alias_name = 4 [deprecated = true]; - - // An alias, which may be a branch or tag. - AliasContext alias_context = 5; - } -} - -// A GitSourceContext denotes a particular revision in a third party Git -// repository (e.g. GitHub). -message GitSourceContext { - // Git repository URL. - string url = 1; - - // Git commit hash. - string revision_id = 2; // required. -} - -// A unique identifier for a cloud repo. -message RepoId { - // A cloud repository can be identified by either its project ID and - // repository name combination, or its globally unique identifier. - oneof id { - // A combination of a project ID and a repo name. - ProjectRepoId project_repo_id = 1; - - // A server-assigned, globally unique identifier. - string uid = 2; - } -} - -// Selects a repo using a Google Cloud Platform project ID -// (e.g. winged-cargo-31) and a repo name within that project. -message ProjectRepoId { - // The ID of the project. - string project_id = 1; - - // The name of the repo. Leave empty for the default repo. - string repo_name = 2; -} - -// A CloudWorkspaceId is a unique identifier for a cloud workspace. -// A cloud workspace is a place associated with a repo where modified files -// can be stored before they are committed. -message CloudWorkspaceId { - // The ID of the repo containing the workspace. - RepoId repo_id = 1; - - // The unique name of the workspace within the repo. This is the name - // chosen by the client in the Source API's CreateWorkspace method. - string name = 2; -} diff --git a/remoteapi/pom.xml b/remoteapi/pom.xml index dcca2df13..7dd4f5fcd 100644 --- a/remoteapi/pom.xml +++ b/remoteapi/pom.xml @@ -56,6 +56,37 @@ com.google.appengine appengine-api-1.0-sdk + + junit + junit + test + + + com.google.truth + truth + test + + + com.google.truth.extensions + truth-proto-extension + 1.4.5 + test + + + org.mockito + mockito-core + test + + + com.google.protobuf + protobuf-java + true + + + org.apache.httpcomponents + httpclient + true + diff --git a/remoteapi/src/main/java/com/google/appengine/tools/remoteapi/RemoteApiInstaller.java b/remoteapi/src/main/java/com/google/appengine/tools/remoteapi/RemoteApiInstaller.java index ca661ae17..51978adb5 100644 --- a/remoteapi/src/main/java/com/google/appengine/tools/remoteapi/RemoteApiInstaller.java +++ b/remoteapi/src/main/java/com/google/appengine/tools/remoteapi/RemoteApiInstaller.java @@ -19,7 +19,6 @@ import com.google.apphosting.api.ApiProxy; import com.google.apphosting.api.ApiProxy.Delegate; import com.google.apphosting.api.ApiProxy.Environment; -import com.google.apphosting.api.ApiProxy.EnvironmentFactory; import com.google.common.collect.ImmutableMap; import java.io.IOException; import java.util.ArrayList; @@ -78,12 +77,14 @@ public class RemoteApiInstaller { private static synchronized StreamHandler getStreamHandler() { if (remoteMethodHandler == null) { remoteMethodHandler = new ConsoleHandler(); - remoteMethodHandler.setFormatter(new Formatter() { - @Override - public String format(LogRecord record) { - return record.getMessage() + "\n"; - } - }); + // Cannot use a lambda here because Formatter is abstract. + remoteMethodHandler.setFormatter( + new Formatter() { + @Override + public String format(LogRecord record) { + return record.getMessage() + "\n"; + } + }); remoteMethodHandler.setLevel(Level.FINE); } return remoteMethodHandler; @@ -142,12 +143,7 @@ public void installOnAllThreads(RemoteApiOptions options) throws IOException { // Single-thread install has the ability to leave the existing environment // in place and simply override the app id. That functionality is not // supported here. - ApiProxy.setEnvironmentFactory(new EnvironmentFactory() { - @Override - public Environment newEnvironment() { - return createEnv(finalOptions, client); - } - }); + ApiProxy.setEnvironmentFactory(() -> createEnv(finalOptions, client)); } } diff --git a/remoteapi/src/main/java/com/google/appengine/tools/remoteapi/RemoteDatastore.java b/remoteapi/src/main/java/com/google/appengine/tools/remoteapi/RemoteDatastore.java index c6fcceeea..a5dd353f3 100644 --- a/remoteapi/src/main/java/com/google/appengine/tools/remoteapi/RemoteDatastore.java +++ b/remoteapi/src/main/java/com/google/appengine/tools/remoteapi/RemoteDatastore.java @@ -23,7 +23,6 @@ import com.google.protobuf.Message; import com.google.storage.onestore.v3_bytes.proto2api.OnestoreEntity; import java.util.ArrayList; -import java.util.Collection; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -195,12 +194,11 @@ static boolean rewriteQueryAppIds(DatastoreV3Pb.Query.Builder query, String remo reserialize = true; query.getAncestorBuilder().setApp(remoteAppId); } - for (DatastoreV3Pb.Query.Filter filter : query.getFilterList()) { - for (OnestoreEntity.Property prop : filter.getPropertyList()) { - OnestoreEntity.PropertyValue propValue = prop.getValue(); - if (propValue.hasReferenceValue()) { + for (DatastoreV3Pb.Query.Filter.Builder filter : query.getFilterBuilderList()) { + for (OnestoreEntity.Property.Builder prop : filter.getPropertyBuilderList()) { + if (prop.getValue().hasReferenceValue()) { OnestoreEntity.PropertyValue.ReferenceValue.Builder ref = - propValue.getReferenceValue().toBuilder(); + prop.getValueBuilder().getReferenceValueBuilder(); if (!ref.getApp().equals(remoteAppId)) { reserialize = true; ref.setApp(remoteAppId); @@ -268,7 +266,7 @@ private byte[] handleGet(byte[] originalRequestBytes) { mergeFromBytes(rewrittenReq, originalRequestBytes); // Update the Request so that all References have the remoteAppId. - boolean reserialize = rewriteRequestReferences(rewrittenReq.getKeyList(), remoteAppId); + boolean reserialize = rewriteRequestReferences(rewrittenReq.getKeyBuilderList(), remoteAppId); if (rewrittenReq.hasTransaction()) { return handleGetWithTransaction(rewrittenReq.build()); } else { @@ -326,7 +324,7 @@ private byte[] handleDelete(byte[] requestBytes) { DatastoreV3Pb.DeleteRequest.Builder request = DatastoreV3Pb.DeleteRequest.newBuilder(); mergeFromBytes(request, requestBytes); - boolean reserialize = rewriteRequestReferences(request.getKeyList(), remoteAppId); + boolean reserialize = rewriteRequestReferences(request.getKeyBuilderList(), remoteAppId); if (reserialize) { // The request was mutated, so we need to reserialize it. requestBytes = request.build().toByteArray(); @@ -348,12 +346,12 @@ private byte[] handleDelete(byte[] requestBytes) { */ /* @VisibleForTesting */ static boolean rewriteRequestReferences( - Collection references, String remoteAppId) { + List references, String remoteAppId) { boolean reserialize = false; - for (OnestoreEntity.Reference refToCheck : references) { + for (OnestoreEntity.Reference.Builder refToCheck : references) { if (!refToCheck.getApp().equals(remoteAppId)) { - refToCheck = refToCheck.toBuilder().setApp(remoteAppId).build(); + refToCheck.setApp(remoteAppId); reserialize = true; } } diff --git a/remoteapi/src/main/java/com/google/appengine/tools/remoteapi/RemoteRpc.java b/remoteapi/src/main/java/com/google/appengine/tools/remoteapi/RemoteRpc.java index 162b86ccb..4544885d7 100644 --- a/remoteapi/src/main/java/com/google/appengine/tools/remoteapi/RemoteRpc.java +++ b/remoteapi/src/main/java/com/google/appengine/tools/remoteapi/RemoteRpc.java @@ -1,4 +1,3 @@ - /* * Copyright 2021 Google LLC * diff --git a/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/OAuthClientTest.java b/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/OAuthClientTest.java new file mode 100644 index 000000000..05246be3b --- /dev/null +++ b/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/OAuthClientTest.java @@ -0,0 +1,122 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.tools.remoteapi; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import com.google.api.client.http.LowLevelHttpRequest; +import com.google.api.client.http.LowLevelHttpResponse; +import com.google.api.client.testing.http.MockHttpTransport; +import com.google.api.client.testing.http.MockLowLevelHttpRequest; +import com.google.api.client.testing.http.MockLowLevelHttpResponse; +import com.google.appengine.tools.remoteapi.AppEngineClient.Response; +import com.google.appengine.tools.remoteapi.testing.StubCredential; +import java.io.IOException; +import java.util.Arrays; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Test for {@link OAuthClient}. + */ +@RunWith(JUnit4.class) +public class OAuthClientTest { + + private static final String APP_ID = "appid"; + + private static class SimpleHttpTransport extends MockHttpTransport { + private final byte[] responseBytes; + + private String lastMethod = null; + private String lastUrl = null; + + SimpleHttpTransport(byte[] responseBytes) { + this.responseBytes = responseBytes; + } + + @Override + public LowLevelHttpRequest buildRequest(String method, String url) throws IOException { + lastMethod = method; + lastUrl = url; + return new MockLowLevelHttpRequest() { + @Override + public LowLevelHttpResponse execute() throws IOException { + MockLowLevelHttpResponse response = new MockLowLevelHttpResponse(); + response.setContent(Arrays.copyOf(responseBytes, responseBytes.length)); + return response; + } + }; + } + + String getLastMethod() { + return lastMethod; + } + + String getLastUrl() { + return lastUrl; + } + } + + @Test + public void testConstructor() { + RemoteApiOptions noOAuthCredential = new RemoteApiOptions() + .credentials("email", "password") + .httpTransport(new SimpleHttpTransport(new byte[] {})); + assertThrows(IllegalArgumentException.class, () -> new OAuthClient(noOAuthCredential, APP_ID)); + + RemoteApiOptions noHttpTransport = new RemoteApiOptions() + .oauthCredential(new StubCredential()); + assertThrows(IllegalStateException.class, () -> new OAuthClient(noHttpTransport, APP_ID)); + } + + @Test + public void testGet() throws Exception { + byte[] expectedResponseBytes = new byte[] {42}; + + SimpleHttpTransport transport = new SimpleHttpTransport(expectedResponseBytes); + RemoteApiOptions options = new RemoteApiOptions() + .server("example.com", 8080) + .oauthCredential(new StubCredential()) + .httpTransport(transport); + + Response response = new OAuthClient(options, APP_ID) + .get("/foo"); + assertTrue(Arrays.equals(expectedResponseBytes, response.getBodyAsBytes())); + assertEquals("GET", transport.getLastMethod()); + assertEquals("http://example.com:8080/foo", transport.getLastUrl()); + } + + @Test + public void testPost() throws Exception { + byte[] expectedResponseBytes = new byte[] {42}; + + SimpleHttpTransport transport = new SimpleHttpTransport(expectedResponseBytes); + RemoteApiOptions options = new RemoteApiOptions() + .server("example.com", 443) + .oauthCredential(new StubCredential()) + .httpTransport(transport); + + Response response = new OAuthClient(options, APP_ID) + .post("/foo", "application/json", new byte[]{1}); + assertTrue(Arrays.equals(expectedResponseBytes, response.getBodyAsBytes())); + assertEquals("POST", transport.getLastMethod()); + assertEquals("https://example.com:443/foo", transport.getLastUrl()); + } +} diff --git a/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/RemoteApiDelegateTest.java b/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/RemoteApiDelegateTest.java new file mode 100644 index 000000000..b70841437 --- /dev/null +++ b/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/RemoteApiDelegateTest.java @@ -0,0 +1,840 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.tools.remoteapi; + +import static com.google.common.io.BaseEncoding.base64Url; +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth.assertWithMessage; +import static com.google.common.truth.extensions.proto.ProtoTruth.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import com.google.appengine.api.datastore.DatastoreService; +import com.google.appengine.api.datastore.DatastoreServiceFactory; +import com.google.appengine.api.datastore.Entity; +import com.google.appengine.api.datastore.EntityNotFoundException; +import com.google.appengine.api.datastore.EntityTranslator; +import com.google.appengine.api.datastore.FetchOptions; +import com.google.appengine.api.datastore.Key; +import com.google.appengine.api.datastore.KeyFactory; +import com.google.appengine.api.datastore.PreparedQuery; +import com.google.appengine.api.datastore.Query; +import com.google.appengine.api.datastore.Transaction; +import com.google.appengine.api.memcache.MemcacheServiceFactory; +import com.google.appengine.api.memcache.MemcacheServicePb; +import com.google.apphosting.api.ApiProxy; +import com.google.apphosting.base.protos.api_bytes.RemoteApiPb; +import com.google.apphosting.datastore_bytes.proto2api.DatastoreV3Pb; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.google.common.hash.Hashing; +import com.google.errorprone.annotations.CanIgnoreReturnValue; +import com.google.protobuf.ByteString; +import com.google.protobuf.ExtensionRegistry; +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.Message; +import com.google.storage.onestore.v3_bytes.proto2api.OnestoreEntity; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.ObjectOutput; +import java.io.ObjectOutputStream; +import java.util.AbstractMap.SimpleImmutableEntry; +import java.util.Iterator; +import java.util.List; +import java.util.Queue; +import java.util.concurrent.LinkedBlockingQueue; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Verifies that the remote API makes the right RPC calls. + * + */ +@RunWith(JUnit4.class) +public class RemoteApiDelegateTest { + private MockRpc rpc; + private DatastoreService datastore; + private RemoteApiInstaller installer; + + @Before + @SuppressWarnings("deprecation") // RemoteApiOptions.credentials is deprecated. + public void setUp() throws Exception { + installer = + new RemoteApiInstaller() { + @Override + AppEngineClient login(RemoteApiOptions options) { + return new AppEngineClient(options, ImmutableList.of(), getAppId()) { + @Override + public Response get(String path) throws IOException { + return null; + } + + @Override + public Response post(String path, String mimeType, byte[] body) throws IOException { + return null; + } + + @Override + public LegacyResponse legacyGet(String path) throws IOException { + return null; + } + + @Override + public LegacyResponse legacyPost(String path, String mimeType, byte[] body) + throws IOException { + return null; + } + }; + } + + @Override + RemoteApiDelegate createDelegate( + RemoteApiOptions options, + RemoteApiClient client, + ApiProxy.Delegate originalDelegate) { + rpc = new MockRpc(client); + return RemoteApiDelegate.newInstance(rpc, options, originalDelegate); + } + + @Override + ApiProxy.Environment createEnv(RemoteApiOptions options, RemoteApiClient client) { + return new ToolEnvironment(getAppId(), "someone@google.com"); + } + }; + + RemoteApiOptions ignoredOptions = + new RemoteApiOptions() + .server("ignored.example.com", 1234) + .credentials("someone@google.com", "ignored"); + installer.install(ignoredOptions); + datastore = DatastoreServiceFactory.getDatastoreService(); + } + + @After + public void tearDown() throws Exception { + installer.uninstall(); + } + + protected String getAppId() { + return "test~appId"; + } + + @Test + public void testFlushMemcache() throws Exception { + rpc.addResponse(MemcacheServicePb.MemcacheFlushResponse.getDefaultInstance()); + + MemcacheServiceFactory.getMemcacheService().clearAll(); + + rpc.verifyNextRpc("memcache", "FlushAll"); + rpc.verifyNoMoreRpc(); + } + + @Test + public void testJavaException() throws Exception { + ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); + ObjectOutput out = new ObjectOutputStream(byteStream); + out.writeObject(new RuntimeException("an exception")); + out.close(); + byte[] serializedException = byteStream.toByteArray(); + + RemoteApiPb.Response response = + RemoteApiPb.Response.newBuilder() + .setJavaException(ByteString.copyFrom(serializedException)) + .build(); + rpc.addResponse(response); + + RuntimeException e = assertThrows(RuntimeException.class, () -> datastore.get(KeyFactory.createKey("Something", 123))); + assertThat(e).hasMessageThat().isEqualTo("an exception"); + } + + @Test + public void testPythonException() throws Exception { + RemoteApiPb.Response response = + RemoteApiPb.Response.newBuilder() + .setException(ByteString.copyFromUtf8("pickle goes here")) + .build(); + rpc.addResponse(response); + + ApiProxy.ApiProxyException e = assertThrows(ApiProxy.ApiProxyException.class, () -> datastore.get(KeyFactory.createKey("Something", 123))); + assertThat(e).hasMessageThat().contains("response was a python exception:\n"); + assertThat(e).hasMessageThat().contains("pickle goes here"); + } + + @Test + public void testQueryReturningAllResultsInOneRpcCall() throws Exception { + replyToQuery(new Entity("Foo"), new Entity("Foo")); + + Query query = new Query("Foo"); + List result = datastore.prepare(query).asList(FetchOptions.Builder.withLimit(10)); + assertEquals(2, result.size()); + + verifyCalledRunQuery("Foo", null); + rpc.verifyNoMoreRpc(); + } + + /** + * Verifies that we can run a query that requires 3 RPC calls. Note that because queries run + * async, the RPC calls overlap with verification! + */ + @Test + public void testQueryReturningResultInThreeRpcCalls() throws Exception { + DatastoreV3Pb.CompiledCursor cursor1 = makeFakeCursor("cursor1"); + DatastoreV3Pb.CompiledCursor cursor2 = makeFakeCursor("cursor2"); + + Query query = new Query("Foo"); + + // Start RPC #1, which runs async. + + replyToQuery(cursor1, new Entity("Foo")); // needed by RPC #1 + Iterator result = datastore.prepare(query).asIterator(); + + // Wait for RPC #1. When it finishes, RPC #2 may start. + + replyToQuery(cursor2, new Entity("Foo"), new Entity("Foo")); // needed by RPC #2 + assertTrue(result.hasNext()); + verifyCalledRunQuery("Foo", null); // check RPC #1 + + // Skip one result from RPC #1 + + result.next(); + + // Wait for RPC #2, and RPC #3 might start. + + replyToQuery(new Entity("Foo")); // needed by RPC #3 + assertTrue(result.hasNext()); + verifyCalledRunQuery("Foo", cursor1); // check RPC #2 (Uses RunQuery, not Next.) + + // Skip two results from RPC #2 + + result.next(); + assertTrue(result.hasNext()); + result.next(); + + // Wait for RPC #3. No more RPC's should start. + + assertTrue(result.hasNext()); + verifyCalledRunQuery("Foo", cursor2); // check RPC #3 + rpc.verifyNoMoreRpc(); + + // Reading the rest of the query results shouldn't require any more RPCs. + + result.next(); + assertFalse(result.hasNext()); + + rpc.verifyNoMoreRpc(); + } + + @Test + public void testTransactionThatInsertsOneEntityWithNewId() throws Exception { + Entity entityToInsert = new Entity("Foo"); + + // Starting a transaction using the remote API doesn't require an RPC. + + Transaction tx = datastore.beginTransaction(); + rpc.verifyNoMoreRpc(); + + // In a transaction, put() calls don't happen immediately, but we need + // an RPC to allocate an id for the new Entity (if it doesn't have one already). + + Key newKey = KeyFactory.createKey("Foo", 123); + replyToGetIds(newKey); + datastore.put(entityToInsert); + rpc.verifyNextRpc("remote_datastore", "GetIDs"); + + assertEquals(newKey, entityToInsert.getKey()); + + // Committing the transaction performs the RPC. + + rpc.verifyNoMoreRpc(); + replyToRemoteTransaction(); + + tx.commit(); + + CommitChecker checker = verifyCommitRequest(); + checker.checkPreconditions(); // none + checker.checkPuts(entityToInsert); + checker.checkDeletes(); // none + } + + @Test + public void testTransactionThatUpdatesOneProperty() throws Exception { + + // The beginning of the transaction is handled locally, without doing any RPC call. + + Transaction tx = datastore.beginTransaction(); + rpc.verifyNoMoreRpc(); + + // When we get() the entity, the RPC is passed through. + + Entity entityReturnedByRpc = new Entity("Foo", "hello"); + replyToGetWithEntity(entityReturnedByRpc); + + Entity entity = datastore.get(KeyFactory.createKey("Foo", "hello")); + + rpc.verifyNextRpc("datastore_v3", "Get"); + assertEquals(entityReturnedByRpc, entity); + + entity.setProperty("bar", 123); + + // The put() call is remembered locally (the datastore won't be updated until commit). + + datastore.put(entity); + rpc.verifyNoMoreRpc(); + + // commit() uses a special RPC call to do the transaction all at once. + + replyToRemoteTransaction(); + + tx.commit(); + + CommitChecker checker = verifyCommitRequest(); + checker.checkPreconditions(entityReturnedByRpc); + checker.checkPuts(entity); + checker.checkDeletes(); // none + + rpc.verifyNoMoreRpc(); + } + + @Test + public void testTransactionThatDeletesAnEntity() throws Exception { + + Key keyToDelete = KeyFactory.createKey("Foo", "hello"); + + Transaction tx = datastore.beginTransaction(); + + // Do the delete. No RPC now, because deletes should be saved until the transaction commits. + + datastore.delete(keyToDelete); + rpc.verifyNoMoreRpc(); + + // Do the commit. + + replyToRemoteTransaction(); + + tx.commit(); + + CommitChecker checker = verifyCommitRequest(); + checker.checkPreconditions(); // none + checker.checkPuts(); // none + checker.checkDeletes(keyToDelete); + } + + @Test + public void testGetsAreCachedInTransaction() throws Exception { + + Key keyToGet = KeyFactory.createKey("Foo", "hello"); + Entity entityReturnedByRpc = new Entity("Foo", "hello"); + + Transaction tx = datastore.beginTransaction(); + + // First get does an rpc + + replyToGetWithEntity(entityReturnedByRpc); + + Entity firstEntity = datastore.get(keyToGet); + + rpc.verifyNextRpc("datastore_v3", "Get"); + assertEquals(entityReturnedByRpc, firstEntity); + + // Second get skips the rpc and returns the same entity + + Entity secondEntity = datastore.get(keyToGet); + rpc.verifyNoMoreRpc(); + + assertNotSame("entities shouldn't be the same", firstEntity, secondEntity); + assertEquals("entities should be equal", firstEntity, secondEntity); + + // The commit should contain one precondition + + replyToRemoteTransaction(); + + tx.commit(); + + CommitChecker checker = verifyCommitRequest(); + checker.checkPreconditions(entityReturnedByRpc); + checker.checkPuts(); // none + checker.checkDeletes(); // none + } + + @Test + public void testTransactionThatGetsNonexistentEntity() throws Exception { + + Key keyToGet = KeyFactory.createKey("Foo", "hello"); + + Transaction tx = datastore.beginTransaction(); + + // The first Get does an rpc and finds out there's no entity in the datastore. + + replyToGetWithMissing(keyToGet); + + EntityNotFoundException e1 = assertThrows(EntityNotFoundException.class, () -> datastore.get(keyToGet)); + assertThat(e1.getKey()).isEqualTo(keyToGet); + + rpc.verifyNextRpc("datastore_v3", "Get"); + + // The second Get skips the rpc and throws the same exception. + + EntityNotFoundException e2 = assertThrows(EntityNotFoundException.class, () -> datastore.get(keyToGet)); + assertThat(e2.getKey()).isEqualTo(keyToGet); + + rpc.verifyNoMoreRpc(); + + // The commit contains a precondition that the entity doesn't exist + + replyToRemoteTransaction(); + + tx.commit(); + + CommitChecker checker = verifyCommitRequest(); + checker.checkNonexistentEntityPreconditions(keyToGet); + checker.checkPuts(); // none + checker.checkDeletes(); // none + } + + @Test + public void testAncestorQueryInTransaction() throws Exception { + Key parentKey = KeyFactory.createKey("Parent", 123); + Key childKey = KeyFactory.createKey(parentKey, "Child", 456); + Entity childEntity = new Entity(childKey); + + Query query = new Query("Child"); + query.setAncestor(parentKey); + + Transaction tx = datastore.beginTransaction(); + PreparedQuery preparedQuery = datastore.prepare(tx, query); + + rpc.verifyNoMoreRpc(); + + Entity witness = replyToTxQuery(null, childEntity); + List result = preparedQuery.asList(FetchOptions.Builder.withLimit(10)); + + assertEquals(1, result.size()); + // The list pulls results lazily, so don't verify the rpc until after we've + // called size(). + verifyCalledTxQuery("Child", null); + assertEquals(childKey, result.get(0).getKey()); + + replyToRemoteTransaction(); + tx.commit(); + + CommitChecker checker = verifyCommitRequest(); + checker.checkPreconditions(witness); + checker.checkPuts(); // none + checker.checkDeletes(); // none + + rpc.verifyNoMoreRpc(); + } + + /** + * Verifies that we can run a query in a transaction that requires 2 RPC calls. Note that because + * queries run async, the RPC calls overlap with verification! + */ + @Test + public void testTwoPartAncestorQueryInTransaction() throws Exception { + DatastoreV3Pb.CompiledCursor cursor1 = makeFakeCursor("cursor1"); + + Key parentKey = KeyFactory.createKey("Parent", 123); + Key child1Key = KeyFactory.createKey(parentKey, "Child", 456); + Entity child1Entity = new Entity(child1Key); + Key child2Key = KeyFactory.createKey(parentKey, "Child", 457); + Entity child2Entity = new Entity(child2Key); + + Query query = new Query("Child"); + query.setAncestor(parentKey); + + Transaction tx = datastore.beginTransaction(); + PreparedQuery preparedQuery = datastore.prepare(tx, query); + + rpc.verifyNoMoreRpc(); + + // start RPC #1 (async) + + Entity witness1 = replyToTxQuery(cursor1, child1Entity); // needed by RPC #1 + Iterator it = preparedQuery.asIterator(); + + // wait for RPC #1 and start RPC #2 + + Entity witness2 = replyToTxQuery(null, child2Entity); // needed by RPC #2 + assertTrue(it.hasNext()); + verifyCalledTxQuery("Child", null); // check RPC #1 + assertEquals(child1Key, it.next().getKey()); + + // wait for RPC #2 + + boolean unused = it.hasNext(); + verifyCalledTxQuery("Child", cursor1); // check RPC #2 + rpc.verifyNoMoreRpc(); + + // check second result + + assertTrue(it.hasNext()); + assertEquals(child2Key, it.next().getKey()); + assertFalse(it.hasNext()); + + rpc.verifyNoMoreRpc(); + + // commit + + replyToRemoteTransaction(); + tx.commit(); + + CommitChecker checker = verifyCommitRequest(); + checker.checkPreconditions(witness1, witness2); + checker.checkPuts(); // none + checker.checkDeletes(); // none + } + + @Test + public void testRollbackTransaction() throws Exception { + Transaction tx = datastore.beginTransaction(); + rpc.verifyNoMoreRpc(); + + tx.rollback(); + rpc.verifyNoMoreRpc(); + } + + @Test + public void testNoRemoteDatastore() throws Exception { + String property = "com.google.appengine.devappserver2"; + String oldProperty = System.getProperty(property); + try { + System.setProperty(property, "true"); + doTestNoRemoteDatastore(); + } finally { + if (oldProperty == null) { + System.clearProperty(property); + } else { + System.setProperty(property, oldProperty); + } + } + } + + @SuppressWarnings( + "deprecation") // RemoteApiOptions.credentials is deprecated, OK to use in tests. + private void doTestNoRemoteDatastore() throws Exception { + RemoteApiOptions remoteApiOptions = + new RemoteApiOptions().credentials("someone@google.com", "password123"); + RemoteApiDelegate delegate = RemoteApiDelegate.newInstance(rpc, remoteApiOptions, null); + ApiProxy.Environment fakeEnvironment = new ToolEnvironment(getAppId(), "someone@google.com"); + byte[] fakeRequest = {1, 2, 3, 4}; + byte[] fakeResponse = {5, 6, 7, 8}; + rpc.addResponse(fakeResponse); + byte[] unused = + delegate.makeSyncCall( + fakeEnvironment, RemoteDatastore.DATASTORE_SERVICE, "Commit", fakeRequest); + rpc.verifyNextRpc(RemoteDatastore.DATASTORE_SERVICE, "Commit"); + rpc.verifyNoMoreRpc(); + } + + // === end of tests === + + @SuppressWarnings("deprecation") // CompiledCursor.Position.start_key is deprecated but we need to + // set it here for testing. + private static DatastoreV3Pb.CompiledCursor makeFakeCursor(String name) { + DatastoreV3Pb.CompiledCursor.Builder result = DatastoreV3Pb.CompiledCursor.newBuilder(); + // DatastoreV3Pb (proto2api) uses String for string fields. + result.getPositionBuilder().setStartKey(ByteString.copyFromUtf8(name)); + return result.build(); + } + + private void replyToQuery(Entity... entities) { + replyToQuery(null, entities); + } + + private void replyToQuery(DatastoreV3Pb.CompiledCursor cursor, Entity... entities) { + DatastoreV3Pb.QueryResult.Builder resultPb = DatastoreV3Pb.QueryResult.newBuilder(); + for (Entity entity : entities) { + resultPb.addResult(entityToProto2(entity)); + } + if (cursor != null) { + resultPb.setMoreResults(true); + resultPb.setCompiledCursor(cursor); + } else { + resultPb.setMoreResults(false); + } + rpc.addResponse(resultPb.build()); + } + + private Entity replyToTxQuery(DatastoreV3Pb.CompiledCursor cursor, Entity... entities) { + RemoteApiPb.TransactionQueryResult.Builder resultPb = + RemoteApiPb.TransactionQueryResult.newBuilder(); + DatastoreV3Pb.QueryResult.Builder queryResultPb = resultPb.getResultBuilder(); + for (Entity entity : entities) { + queryResultPb.addResult(entityToProto2(entity)); + } + if (cursor != null) { + queryResultPb.setMoreResults(true); + queryResultPb.setCompiledCursor(cursor); + } else { + queryResultPb.setMoreResults(false); + } + // Make a somewhat arbitrary witness. + Entity witness = new Entity("__entity_group__", "foo", entities[0].getKey()); + resultPb + .setEntityGroupKey(entityToProto2(witness).getKey()) + .setEntityGroup(entityToProto2(witness)); + rpc.addResponse(resultPb.build()); + return witness; + } + + private void replyToGetWithMissing(Key missingKey) { + DatastoreV3Pb.GetResponse.Builder response = DatastoreV3Pb.GetResponse.newBuilder(); + response.addEntityBuilder().setKey(keyToRefProto2(missingKey)); + rpc.addResponse(response.build()); + } + + private void replyToGetWithEntity(Entity entity) { + DatastoreV3Pb.GetResponse.Builder response = DatastoreV3Pb.GetResponse.newBuilder(); + response.addEntityBuilder().setEntity(entityToProto2(entity)); + rpc.addResponse(response.build()); + } + + private void replyToGetIds(Key... updatedKeys) { + DatastoreV3Pb.PutResponse.Builder response = DatastoreV3Pb.PutResponse.newBuilder(); + for (Key key : updatedKeys) { + response.addKey(keyToRefProto2(key)); + } + rpc.addResponse(response.build()); + } + + private void replyToRemoteTransaction() { + DatastoreV3Pb.PutResponse response = DatastoreV3Pb.PutResponse.getDefaultInstance(); + rpc.addResponse(response.toByteArray()); + } + + @CanIgnoreReturnValue + private DatastoreV3Pb.Query verifyCalledRunQuery( + String expectedKind, DatastoreV3Pb.CompiledCursor expectedCursor) { + RemoteApiPb.Request wrappedRequest = rpc.verifyNextRpc("datastore_v3", "RunQuery"); + DatastoreV3Pb.Query query = verifyQuery(wrappedRequest, expectedKind, expectedCursor); + // not in a transaction + assertFalse("query shouldn't be in a transaction", query.hasTransaction()); + + return query; + } + + @CanIgnoreReturnValue + private DatastoreV3Pb.Query verifyCalledTxQuery( + String expectedKind, DatastoreV3Pb.CompiledCursor expectedCursor) { + RemoteApiPb.Request wrappedRequest = rpc.verifyNextRpc("remote_datastore", "TransactionQuery"); + DatastoreV3Pb.Query query = verifyQuery(wrappedRequest, expectedKind, expectedCursor); + + return query; + } + + @SuppressWarnings("deprecation") // Testing deprecated Query.compile field. + private static DatastoreV3Pb.Query verifyQuery( + RemoteApiPb.Request wrappedRequest, + String expectedKind, + DatastoreV3Pb.CompiledCursor expectedCursor) { + DatastoreV3Pb.Query query; + try { + query = + DatastoreV3Pb.Query.parseFrom( + wrappedRequest.getRequest(), ExtensionRegistry.getEmptyRegistry()); + } catch (InvalidProtocolBufferException e) { + throw new RuntimeException(e); + } + + assertEquals("query on unexpected kind", expectedKind, query.getKind()); + assertWithMessage("Compile flag not set for query").that(query.getCompile()).isTrue(); + assertEquals("query with unexpected offset", 0, query.getOffset()); + if (expectedCursor != null) { + assertEquals("cursor doesn't match", expectedCursor, query.getCompiledCursor()); + } else { + assertFalse(query.hasCompiledCursor()); + } + + return query; + } + + private CommitChecker verifyCommitRequest() { + RemoteApiPb.Request wrappedRequest = rpc.verifyNextRpc("remote_datastore", "Transaction"); + return new CommitChecker(wrappedRequest); + } + + private static OnestoreEntity.EntityProto entityToProto2(Entity entity) { + return EntityTranslator.convertToPb(entity); + } + + private static OnestoreEntity.Reference keyToRefProto2(Key key) { + try { + String encoded = KeyFactory.keyToString(key); + byte[] bytes = base64Url().decode(encoded); + return OnestoreEntity.Reference.parseFrom(bytes, ExtensionRegistry.getEmptyRegistry()); + } catch (Exception e) { + throw new RuntimeException("can't convert key to reference", e); + } + } + + /** + * Accepts RPC calls and returns the appropriate fake value. Handles async calls by waiting before + * verifying. + */ + static class MockRpc extends RemoteRpc { + + Queue receivedRequests; + Queue responsesToSend; + + MockRpc(RemoteApiClient client) { + super(client); + receivedRequests = new LinkedBlockingQueue(); + responsesToSend = new LinkedBlockingQueue(); + } + + void addResponse(RemoteApiPb.Response response) { + responsesToSend.add(response); + } + + void addResponse(byte[] bytes) { + RemoteApiPb.Response response = + RemoteApiPb.Response.newBuilder().setResponse(ByteString.copyFrom(bytes)).build(); + + addResponse(response); + } + + void addResponse(Message result) { + addResponse(result.toByteArray()); + } + + @Override + RemoteApiPb.Response callImpl(RemoteApiPb.Request requestProto) { + receivedRequests.add(requestProto); + return responsesToSend.remove(); + } + + /** + * Checks that at least one RPC happened and returns its value. In the case of async RPC, you + * must arrange to wait until the async call finishes before calling this method (typically by + * getting the value from the Future). + */ + RemoteApiPb.Request verifyNextRpc() { + return receivedRequests.remove(); + } + + @CanIgnoreReturnValue + RemoteApiPb.Request verifyNextRpc(String expectedService, String expectedMethod) { + RemoteApiPb.Request request = verifyNextRpc(); + assertWithMessage("unexpected method call") + .that(request.getServiceName() + "." + request.getMethod()) + .isEqualTo(expectedService + "." + expectedMethod); + return request; + } + + /** Verify that no RPC happened. (Waits a bit to make sure an async RPC isn't in progress.) */ + void verifyNoMoreRpc() throws InterruptedException { + // need to wait in the case of async calls + Thread.sleep(1000); + assertThat(receivedRequests).isEmpty(); + assertThat(responsesToSend).isEmpty(); + } + } + + /** Verifies the fields in a request to commit a transaction using the remote API. */ + static class CommitChecker { + private final RemoteApiPb.TransactionRequest actualRequest; + + CommitChecker(RemoteApiPb.Request wrappedRequest) { + try { + actualRequest = + RemoteApiPb.TransactionRequest.parseFrom( + wrappedRequest.getRequest(), ExtensionRegistry.getEmptyRegistry()); + } catch (InvalidProtocolBufferException e) { + throw new RuntimeException(e); + } + } + + void checkPreconditions(Entity... expectedEntities) { + + List> expected = Lists.newArrayList(); + for (Entity entity : expectedEntities) { + ByteString key = entityToProto2(entity).getKey().toByteString(); + ByteString hash = getSha1Hash(entity); + expected.add(new SimpleImmutableEntry<>(key, hash)); + } + + checkPreconditions(expected); + } + + private void checkPreconditions(List> expected) { + List> actual = Lists.newArrayList(); + for (RemoteApiPb.TransactionRequest.Precondition precondition : + actualRequest.getPreconditionList()) { + ByteString key = precondition.getKey().toByteString(); + ByteString hash = precondition.hasHash() ? precondition.getHash() : null; + actual.add(new SimpleImmutableEntry<>(key, hash)); + } + + assertWithMessage("commit call doesn't have the right preconditions") + .that(actual) + .containsExactlyElementsIn(expected); + } + + void checkNonexistentEntityPreconditions(Key... expectedKeys) { + List> expected = Lists.newArrayList(); + for (Key key : expectedKeys) { + ByteString keyBytes = keyToRefProto2(key).toByteString(); + expected.add(new SimpleImmutableEntry<>(keyBytes, (ByteString) null)); + } + checkPreconditions(expected); + } + + void checkPuts(Entity... expectedPuts) { + assertWithMessage("commit call doesn't put() the right entities") + .that(createEntities(actualRequest.getPuts().getEntityList())) + .containsExactlyElementsIn(expectedPuts); + } + + void checkDeletes(Key... expected) { + assertWithMessage("commit call doesn't delete the right entities") + .that(createKeys(actualRequest.getDeletes().getKeyList())) + .containsExactlyElementsIn(expected); + } + + private static ByteString getSha1Hash(Entity entity) { + byte[] bytes = entityToProto2(entity).toByteArray(); + return ByteString.copyFrom(Hashing.sha1().hashBytes(bytes, 0, bytes.length).asBytes()); + } + + private static List createEntities(Iterable entities) { + List result = Lists.newArrayList(); + for (OnestoreEntity.EntityProto entityPb : entities) { + result.add(EntityTranslator.createFromPb(entityPb)); + } + return result; + } + + private static List createKeys(Iterable keys) { + List result = Lists.newArrayList(); + for (OnestoreEntity.Reference keyProto : keys) { + result.add(createKey(keyProto)); + } + return result; + } + + private static Key createKey(OnestoreEntity.Reference keyPb) { + // KeyTranslator isn't public but we can call it indirectly through KeyFactory + return KeyFactory.stringToKey(base64Url().omitPadding().encode(keyPb.toByteArray())); + } + } +} diff --git a/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/RemoteApiInstallerAllThreadsTest.java b/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/RemoteApiInstallerAllThreadsTest.java new file mode 100644 index 000000000..481027c86 --- /dev/null +++ b/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/RemoteApiInstallerAllThreadsTest.java @@ -0,0 +1,102 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.tools.remoteapi; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; + +import com.google.apphosting.api.ApiProxy; +import java.io.IOException; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import org.apache.http.cookie.Cookie; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Test for {@link RemoteApiInstaller#installOnAllThreads(RemoteApiOptions)}. + * This method leaves behind static state in {@link ApiProxy} that can't be + * changed, so it's run as a separate test. + */ +@RunWith(JUnit4.class) +public class RemoteApiInstallerAllThreadsTest { + + @Test + public void testInstallOnAllThreads() throws Exception { + // Confirm that we're starting with a clean state. + assertNull(ApiProxy.getCurrentEnvironment()); + assertNull(ApiProxy.getDelegate()); + + final RemoteApiInstaller installer = newInstaller("appId"); + installer.installOnAllThreads(createDummyOptions()); + + // Inspect the ApiProxy from this thread. + assertNotNull(ApiProxy.getDelegate()); + assertNotNull(ApiProxy.getCurrentEnvironment()); + + ExecutorService executor = Executors.newSingleThreadExecutor(); + + // Inspect the ApiProxy from another thread. + executor.submit(() -> { + assertNotNull(ApiProxy.getDelegate()); + assertNotNull(ApiProxy.getCurrentEnvironment()); + }).get(); + + // Can't re-install on all threads. + assertThrows(IllegalStateException.class, () -> installer.installOnAllThreads(createDummyOptions())); + + // Can't re-install on all threads from a separate thread. + executor.submit(() -> { + assertThrows(IllegalStateException.class, () -> installer.installOnAllThreads(createDummyOptions())); + }).get(); + + // Can't install for a single thread. + assertThrows(IllegalStateException.class, () -> installer.install(createDummyOptions())); + + // Can't install for a single separate thread. + executor.submit(() -> { + assertThrows(IllegalStateException.class, () -> installer.install(createDummyOptions())); + }).get(); + + // Can't uninstall. + assertThrows(IllegalArgumentException.class, () -> installer.uninstall()); + + // Can't uninstall from a separate thread. + executor.submit(() -> { + assertThrows(IllegalArgumentException.class, () -> installer.uninstall()); + }).get(); + } + + private static RemoteApiOptions createDummyOptions() { + return new RemoteApiOptions() + .server("localhost", 8080) + .credentials("this", "that"); + } + + private static RemoteApiInstaller newInstaller(final String remoteAppId) { + return new RemoteApiInstaller() { + @Override + String getAppIdFromServer(List authCookies, RemoteApiOptions options) + throws IOException { + return remoteAppId; + } + }; + } +} diff --git a/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/RemoteApiInstallerTest.java b/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/RemoteApiInstallerTest.java new file mode 100644 index 000000000..12402673e --- /dev/null +++ b/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/RemoteApiInstallerTest.java @@ -0,0 +1,325 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.tools.remoteapi; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import com.google.appengine.tools.remoteapi.testing.StubCredential; +import com.google.apphosting.api.ApiProxy; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import org.apache.http.cookie.Cookie; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Verifies some of the methods in {@link RemoteApiInstaller}. + * + */ +@RunWith(JUnit4.class) +public class RemoteApiInstallerTest { + + @Before + public void setUp() throws Exception { + // Start with no environment (simulating a standalone setup). Individual tests may change this. + ApiProxy.setEnvironmentForCurrentThread(null); + } + + /** + * Verifies that we can parse the YAML map returned by the RemoteApiServlet. + * (This map contains the app id.) + */ + @Test + public void testParseYamlMap() throws Exception { + Map output = RemoteApiInstaller.parseYamlMap( + " {rtok: null, app_id: my-app-id}\n\n"); + + assertEquals("null", output.get("rtok")); + assertEquals("my-app-id", output.get("app_id")); + assertThat(output).hasSize(2); + + // now try an HR app + output = RemoteApiInstaller.parseYamlMap( + " {rtok: null, app_id: s~my-app-id}\n\n"); + + assertEquals("null", output.get("rtok")); + assertEquals("s~my-app-id", output.get("app_id")); + assertThat(output).hasSize(2); + + // the fields may be quoted using double strings. + output = RemoteApiInstaller.parseYamlMap( + "{app_id: \"s~go-tee\", rtok: \"0\"}"); + + assertEquals("s~go-tee", output.get("app_id")); + assertEquals("0", output.get("rtok")); + assertThat(output).hasSize(2); + + // mismatched quotes should fail to parse. + output = RemoteApiInstaller.parseYamlMap( + "{app_id: 's~go-tee\", rtok: \"0}"); + + assertThat(output).isEmpty(); + + output = RemoteApiInstaller.parseYamlMap( + "{app_id: s~go-tee', rtok: \"0'}"); + + assertThat(output).isEmpty(); + } + + /** + * Verifies that we can parse the YAML map returned by the RemoteApiServlet + * for an app running on internal infrastructure. + * (This map contains the app id.) + */ + @Test + public void testParseYamlMapInternal() throws Exception { + Map output = RemoteApiInstaller.parseYamlMap( + " {rtok: 'null', app_id: 'google.com:my-app-id'}\n\n"); + + assertEquals("null", output.get("rtok")); + assertEquals("google.com:my-app-id", output.get("app_id")); + assertThat(output).hasSize(2); + + // now try an HR app + output = RemoteApiInstaller.parseYamlMap( + " {rtok: 'null', app_id: 's~google.com:my-app-id'}\n\n"); + + assertEquals("null", output.get("rtok")); + assertEquals("s~google.com:my-app-id", output.get("app_id")); + assertThat(output).hasSize(2); + } + + @Test + public void testParseSerializedCredentials() throws Exception { + String credentials = + """ + + # example credential file + + host=somehost.prom.corp.google.com + email=foo@google.com + + cookie=SACSID=ASDFASDF + cookie=SOMETHING=else + """; + + List cookies = RemoteApiInstaller.parseSerializedCredentials("foo@google.com", + "somehost.prom.corp.google.com", credentials); + + Cookie cookie = cookies.get(0); + assertEquals("SACSID", cookie.getName()); + assertEquals("ASDFASDF", cookie.getValue()); + + cookie = cookies.get(1); + assertEquals("SOMETHING", cookie.getName()); + assertEquals("else", cookie.getValue()); + + assertEquals(2, cookies.size()); + } + + @Test + @SuppressWarnings("rawtypes") // ApiProxy.getDelegate() returns a raw type. + public void testInstallationStandalone() throws Exception { + assertNull(ApiProxy.getCurrentEnvironment()); + + // On Install, it should set a ToolsEnvironment on the Thread. + RemoteApiInstaller installer = install("yar"); + assertTrue(ToolEnvironment.class.equals(ApiProxy.getCurrentEnvironment().getClass())); + assertThat(ApiProxy.getDelegate()).isInstanceOf(ThreadLocalDelegate.class); + + // Removes the Environment on uninstall. + installer.uninstall(); + assertNull(ApiProxy.getCurrentEnvironment()); + assertThat(ApiProxy.getDelegate()).isInstanceOf(ThreadLocalDelegate.class); + ThreadLocalDelegate tld = (ThreadLocalDelegate) ApiProxy.getDelegate(); + + installer = install("yar"); + assertSame(tld, ApiProxy.getDelegate()); + installer.uninstall(); + assertSame(tld, ApiProxy.getDelegate()); + } + + @Test + public void testInstallationHosted() throws Exception { + // This is kind of cheating. The RemoteApiInstaller checks for the presence of an Environment + // to determine if it's running standalone or hosted. We'll trick it by putting a + // ToolsEnvironment in there. + ApiProxy.setEnvironmentForCurrentThread(new ToolEnvironment("unused-appid", "unused-email")); + + RemoteApiInstaller installer = install("yar"); + + // Should have left the existing Environment in place, but added an attribute to override the + // app ids in Datastore keys. + assertTrue(ToolEnvironment.class.equals(ApiProxy.getCurrentEnvironment().getClass())); + assertEquals( + "yar", + ApiProxy.getCurrentEnvironment().getAttributes().get( + RemoteApiInstaller.DATASTORE_APP_ID_OVERRIDE_KEY)); + + // The app id override is removed after uninstalling. + installer.uninstall(); + assertThat(ApiProxy.getCurrentEnvironment().getAttributes()) + .doesNotContainKey(RemoteApiInstaller.DATASTORE_APP_ID_OVERRIDE_KEY); + } + + @Test + public void testInstallationHostedWithExistingAppIdOverride() throws Exception { + // This is kind of cheating. The RemoteApiInstaller checks for the presence of an Environment + // to determine if it's running standalone or hosted. We'll trick it by putting a + // ToolsEnvironment in there. + ApiProxy.setEnvironmentForCurrentThread(new ToolEnvironment("unused-appid", "unused-email")); + + // Simulate there already being an override. This is not something we really expect, but we'll + // make sure it works. + ApiProxy.getCurrentEnvironment().getAttributes().put( + RemoteApiInstaller.DATASTORE_APP_ID_OVERRIDE_KEY, "somePreexistingOverride"); + + RemoteApiInstaller installer = install("yar"); + + // Should have left the existing Environment in place, but changed the app id override. + assertTrue(ToolEnvironment.class.equals(ApiProxy.getCurrentEnvironment().getClass())); + assertEquals( + "yar", + ApiProxy.getCurrentEnvironment().getAttributes().get( + RemoteApiInstaller.DATASTORE_APP_ID_OVERRIDE_KEY)); + + // The app id override is removed after uninstalling and should restore the old override. + installer.uninstall(); + assertEquals( + "somePreexistingOverride", + ApiProxy.getCurrentEnvironment().getAttributes().get( + RemoteApiInstaller.DATASTORE_APP_ID_OVERRIDE_KEY)); + } + + @Test + @SuppressWarnings("rawtypes") + public void testInstallationOnDifferentThreads() throws Exception { + // Install in the test thread. + RemoteApiInstaller installer1 = install("yar"); + final ApiProxy.Delegate tld = ApiProxy.getDelegate(); + assertThat(tld).isInstanceOf(ThreadLocalDelegate.class); + + ExecutorService svc = Executors.newSingleThreadExecutor(); + + // Install in an alternate thread. + Callable callable = () -> { + RemoteApiInstaller installer = install("yar"); + assertSame(tld, ApiProxy.getDelegate()); + assertNotNull(((ThreadLocalDelegate) ApiProxy.getDelegate()).getDelegateForThread()); + return installer; + }; + final RemoteApiInstaller installer2 = svc.submit(callable).get(); + + // Uninstall in the test thread. + installer1.uninstall(); + assertSame(tld, ApiProxy.getDelegate()); + assertNull(((ThreadLocalDelegate) ApiProxy.getDelegate()).getDelegateForThread()); + + // Uninstall in the alternate thread. + @SuppressWarnings("unused") // go/futurereturn-lsc + Future possiblyIgnoredError = + svc.submit( + () -> { + assertThat(((ThreadLocalDelegate) ApiProxy.getDelegate()).getDelegateForThread()) + .isNotNull(); + installer2.uninstall(); + assertThat(((ThreadLocalDelegate) ApiProxy.getDelegate()).getDelegateForThread()) + .isNull(); + return null; + }); + assertSame(tld, ApiProxy.getDelegate()); + assertNull(((ThreadLocalDelegate) ApiProxy.getDelegate()).getDelegateForThread()); + } + + @Test + public void testValidateOptionsNullHostname() { + RemoteApiOptions options = new RemoteApiOptions() + .server(null, 8080) + .credentials("email", "password"); + assertThrows(IllegalArgumentException.class, () -> new RemoteApiInstaller().validateOptions(options)); + } + + @Test + public void testValidateOptionsNoCredentials() { + RemoteApiOptions options = new RemoteApiOptions() + .server("hostname", 8080); + assertThrows(IllegalArgumentException.class, () -> new RemoteApiInstaller().validateOptions(options)); + } + + @Test + public void testValidateOptionsPasswordCredentials() { + RemoteApiOptions options = new RemoteApiOptions() + .server("hostname", 8080) + .credentials("email", "password"); + new RemoteApiInstaller().validateOptions(options); + } + + @Test + public void testValidateOptionsDevAppServerCredentials() { + RemoteApiOptions options = new RemoteApiOptions() + .server("hostname", 8080) + .useDevelopmentServerCredential(); + new RemoteApiInstaller().validateOptions(options); + } + + @Test + public void testValidateOptionsOAuthCredentials() { + RemoteApiOptions options = new RemoteApiOptions() + .server("hostname", 8080) + .oauthCredential(new StubCredential()); + new RemoteApiInstaller().validateOptions(options); + } + + private static RemoteApiOptions createDummyOptions() { + return new RemoteApiOptions() + .server("localhost", 8080) + .credentials("this", "that"); + } + + private static RemoteApiInstaller newInstaller(final String remoteAppId) { + return new RemoteApiInstaller() { + @Override + String getAppIdFromServer(List authCookies, RemoteApiOptions options) + throws IOException { + return remoteAppId; + } + }; + } + + RemoteApiInstaller install(final String remoteAppId) { + RemoteApiInstaller installer = newInstaller(remoteAppId); + try { + installer.install(createDummyOptions()); + } catch (IOException e) { + throw new RuntimeException(e); + } + return installer; + } +} diff --git a/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/RemoteApiOptionsTest.java b/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/RemoteApiOptionsTest.java new file mode 100644 index 000000000..1abf9230c --- /dev/null +++ b/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/RemoteApiOptionsTest.java @@ -0,0 +1,213 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.tools.remoteapi; + +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotSame; + +import com.google.api.client.testing.http.MockHttpTransport; +import com.google.appengine.tools.remoteapi.testing.StubCredential; +import com.google.apphosting.api.ApiProxy; +import com.google.apphosting.api.ApiProxy.Environment; +import com.google.common.collect.ImmutableList; +import java.io.File; +import java.io.InputStream; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.net.URL; +import java.nio.file.Files; +import java.util.Map; +import java.util.Objects; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Tests for {@link RemoteApiOptions}. In particular this ensures that if we add a new option we + * remember to update the copy method. + * + */ +@RunWith(JUnit4.class) +public class RemoteApiOptionsTest { + + private static class StubEnvironment implements Environment { + @Override + public boolean isLoggedIn() { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isAdmin() { + throw new UnsupportedOperationException(); + } + + @Override + public String getVersionId() { + throw new UnsupportedOperationException(); + } + + @Deprecated + @Override + public String getRequestNamespace() { + throw new UnsupportedOperationException(); + } + + @Override + public long getRemainingMillis() { + throw new UnsupportedOperationException(); + } + + @Override + public String getModuleId() { + throw new UnsupportedOperationException(); + } + + @Override + public String getEmail() { + throw new UnsupportedOperationException(); + } + + @Override + public String getAuthDomain() { + throw new UnsupportedOperationException(); + } + + @Override + public Map getAttributes() { + throw new UnsupportedOperationException(); + } + + @Override + public String getAppId() { + throw new UnsupportedOperationException(); + } + } + + /** + * Return a list of RemoteApiOptions objects such that for any given property, at least one of + * the objects has a non-default value for that property. + * This will need to be updated every time a new property is added to the class. + */ + private static ImmutableList nonDefaultOptions() { + RemoteApiOptions optionsWithPassword = new RemoteApiOptions() + .credentials("foo@bar.com", "mysecurepassword") + .datastoreQueryFetchSize(17) + .maxConcurrentRequests(23) + .maxHttpResponseSize(1729) + .remoteApiPath("/path/to/remote/api") + .server("somehostname", 8080); + RemoteApiOptions optionsWithCredentials = optionsWithPassword.copy() + .reuseCredentials("foo@bar.com", "myserializedcredentials"); + RemoteApiOptions optionsWithOAuthCredentials = optionsWithPassword.copy() + .oauthCredential(new StubCredential()) + .httpTransport(new MockHttpTransport()); + return ImmutableList.of(optionsWithPassword, optionsWithCredentials, + optionsWithOAuthCredentials); + } + + /** + * Test that {@link #nonDefaultOptions()} does return a non-default value somewhere for every + * property. The validity of the tests for {@link RemoteApiOptions#copy} depends on this. + */ + @Test + public void testNonDefaultOptions() throws Exception { + RemoteApiOptions defaultOptions = new RemoteApiOptions(); + for (Field field : RemoteApiOptions.class.getDeclaredFields()) { + if (!Modifier.isStatic(field.getModifiers())) { + boolean allSame = true; + for (RemoteApiOptions nonDefaultOptions : nonDefaultOptions()) { + allSame &= sameValueOfFieldIn(field, defaultOptions, nonDefaultOptions); + } + assertFalse(field.getName(), allSame); + } + } + } + + @Test + public void testDefaultCopy() { + RemoteApiOptions defaultOptions = new RemoteApiOptions(); + assertOptionsEqual(defaultOptions, defaultOptions.copy()); + } + + @Test + public void testCopyCopiesEverything() throws Exception { + for (RemoteApiOptions nonDefaultOptions : nonDefaultOptions()) { + RemoteApiOptions copy = nonDefaultOptions.copy(); + assertOptionsEqual(nonDefaultOptions, copy); + } + } + + @Test + public void testOAuthCredentialsSupportedOnAppEngineClient() throws Exception { + ApiProxy.setEnvironmentForCurrentThread(new StubEnvironment()); + URL url = + this.getClass() + .getClassLoader() + .getResource("com/google/appengine/tools/remoteapi/testdata/test.pkcs12"); + if (url == null) { + url = + this.getClass() + .getClassLoader() + .getResource( + "src/test/resources/com/google/appengine/tools/remoteapi/testdata/test.pkcs12"); + } + if (url == null) { + url = + this.getClass() + .getClassLoader() + .getResource( + "third_party/java_src/appengine_standard/remoteapi/src/test/resources/com/google/appengine/tools/remoteapi/testdata/test.pkcs12"); + } + File tempFile = File.createTempFile("test", ".pkcs12"); + tempFile.deleteOnExit(); + try (InputStream in = url.openStream()) { + Files.copy(in, tempFile.toPath(), REPLACE_EXISTING); + } + RemoteApiOptions options = new RemoteApiOptions(); + options.useServiceAccountCredential("foo@example.com", tempFile.getAbsolutePath()); + } + + private static void assertOptionsEqual(RemoteApiOptions x, RemoteApiOptions y) { + assertNotSame(x, y); + assertEquals(null, differingOption(x, y)); + // JUnit doesn't show the non-null value if assertNull fails. Grrr. + } + + /** + * Return the first option that differs between the two objects, or null if they have all the + * same options. + */ + private static String differingOption(RemoteApiOptions x, RemoteApiOptions y) { + for (Field field : RemoteApiOptions.class.getDeclaredFields()) { + if (!sameValueOfFieldIn(field, x, y)) { + return field.getName(); + } + } + return null; + } + + private static boolean sameValueOfFieldIn(Field field, RemoteApiOptions x, RemoteApiOptions y) { + try { + field.setAccessible(true); + return Objects.deepEquals(field.get(x), field.get(y)); + } catch (IllegalAccessException e) { + throw new AssertionError(e); + } + } +} diff --git a/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/RemoteDatastoreTest.java b/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/RemoteDatastoreTest.java new file mode 100644 index 000000000..820b43498 --- /dev/null +++ b/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/RemoteDatastoreTest.java @@ -0,0 +1,454 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.tools.remoteapi; + +import static com.google.common.truth.Truth.assertWithMessage; +import static com.google.common.truth.extensions.proto.ProtoTruth.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.when; + +import com.google.apphosting.datastore_bytes.proto2api.DatastoreV3Pb; +import com.google.common.collect.Lists; +import com.google.protobuf.ByteString; +import com.google.protobuf.ExtensionRegistry; +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.storage.onestore.v3_bytes.proto2api.OnestoreEntity; +import com.google.storage.onestore.v3_bytes.proto2api.OnestoreEntity.EntityProto; +import com.google.storage.onestore.v3_bytes.proto2api.OnestoreEntity.Reference; +import java.util.List; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +/** + * Test for {@link RemoteDatastore} + * + * @author maxr@google.com (Max Ross) + */ +@RunWith(JUnit4.class) +public class RemoteDatastoreTest { + + private static final String CLIENT_APP_ID = "client"; + private static final String TARGET_APP_ID = "target"; + + @Rule public final MockitoRule mocks = MockitoJUnit.rule(); + @Mock private RemoteRpc mockRemoteRpc; + + private RemoteDatastore remoteDatastore; + + private static final RewrittenReferenceHolder REFERENCE_HOLDER_1 = + new RewrittenReferenceHolder(1); + private static final RewrittenReferenceHolder REFERENCE_HOLDER_2 = + new RewrittenReferenceHolder(2); + private static final RewrittenReferenceHolder REFERENCE_HOLDER_3 = + new RewrittenReferenceHolder(3); + private static final RewrittenReferenceHolder REFERENCE_HOLDER_4 = + new RewrittenReferenceHolder(4); + private static final RewrittenReferenceHolder REFERENCE_HOLDER_5 = + new RewrittenReferenceHolder(5); + private static final RewrittenReferenceHolder REFERENCE_HOLDER_6 = + new RewrittenReferenceHolder(6); + + /** Wrapper around both formats of Reference and Entity. */ + private static class RewrittenReferenceHolder { + private final Reference clientReference; + private final Reference targetReference; + private final EntityProto clientEntity; + private final EntityProto targetEntity; + + private RewrittenReferenceHolder(int index) { + this.clientEntity = createEntityProto(CLIENT_APP_ID, index); + this.targetEntity = createEntityProto(TARGET_APP_ID, index); + this.clientReference = clientEntity.getKey(); + this.targetReference = targetEntity.getKey(); + } + } + + @Before + public void setUpRemoteDatastore() { + remoteDatastore = new RemoteDatastore(TARGET_APP_ID, mockRemoteRpc, new RemoteApiOptions()); + } + + @Test + public void testGetRewritesReferences() throws InvalidProtocolBufferException { + DatastoreV3Pb.GetRequest.Builder getRequest = DatastoreV3Pb.GetRequest.newBuilder(); + getRequest.addKey(REFERENCE_HOLDER_1.clientReference); + getRequest.addKey(REFERENCE_HOLDER_2.clientReference); + getRequest.addKey(REFERENCE_HOLDER_3.clientReference); + getRequest.addKey(REFERENCE_HOLDER_4.targetReference); // This key already has the target app. + getRequest.addKey(REFERENCE_HOLDER_5.clientReference); + + DatastoreV3Pb.GetRequest rewrittenGetRequest = + DatastoreV3Pb.GetRequest.newBuilder() + .addKey(REFERENCE_HOLDER_1.targetReference) + .addKey(REFERENCE_HOLDER_2.targetReference) + .addKey(REFERENCE_HOLDER_3.targetReference) + .addKey(REFERENCE_HOLDER_4.targetReference) + .addKey(REFERENCE_HOLDER_5.targetReference) + .build(); + + // A mix of Entities, Missing, and Deferred. + DatastoreV3Pb.GetResponse.Builder remoteRpcResponse = DatastoreV3Pb.GetResponse.newBuilder(); + remoteRpcResponse.addEntityBuilder().setKey(REFERENCE_HOLDER_1.targetReference); + remoteRpcResponse.addEntityBuilder().setEntity(REFERENCE_HOLDER_2.targetEntity); + remoteRpcResponse.addEntityBuilder().setEntity(REFERENCE_HOLDER_3.targetEntity); + remoteRpcResponse.addDeferred(REFERENCE_HOLDER_4.targetReference); + remoteRpcResponse.addEntityBuilder().setKey(REFERENCE_HOLDER_5.targetReference); + + expectRemoteRpcGet(rewrittenGetRequest, remoteRpcResponse.build()); + + // We should get a serialized version of the remoteRpcResponse. + DatastoreV3Pb.GetResponse actualResp = invokeGet(getRequest.build()); + assertThat(actualResp).isEqualTo(remoteRpcResponse.build()); + } + + @Test + public void testGetAndPutInTransaction() throws InvalidProtocolBufferException { + // Begin a transaction. + DatastoreV3Pb.Transaction transaction = beginTransaction(); + + // Issue a Get request. + DatastoreV3Pb.GetRequest getRequest1 = + DatastoreV3Pb.GetRequest.newBuilder() + .setTransaction(transaction) + .addKey(REFERENCE_HOLDER_1.clientReference) + .addKey(REFERENCE_HOLDER_2.clientReference) + .build(); + + DatastoreV3Pb.GetRequest rewrittenGetRequest1 = + DatastoreV3Pb.GetRequest.newBuilder() + .addKey(REFERENCE_HOLDER_1.targetReference) + .addKey(REFERENCE_HOLDER_2.targetReference) + .build(); + + // One found, one missing + DatastoreV3Pb.GetResponse.Builder remoteRpcGetResponse1 = + DatastoreV3Pb.GetResponse.newBuilder(); + remoteRpcGetResponse1.addEntityBuilder().setKey(REFERENCE_HOLDER_1.targetReference); + remoteRpcGetResponse1.addEntityBuilder().setEntity(REFERENCE_HOLDER_2.targetEntity); + + expectRemoteRpcGet(rewrittenGetRequest1, remoteRpcGetResponse1.build()); + + // We should get a serialized version of the remoteRpcResponse. + DatastoreV3Pb.GetResponse actualResp1 = invokeGet(getRequest1); + assertThat(actualResp1) + .ignoringFields(DatastoreV3Pb.GetResponse.IN_ORDER_FIELD_NUMBER) + .isEqualTo(remoteRpcGetResponse1.build()); + + // Now do a Put. + EntityProto.Builder mutatedClientEntity2 = REFERENCE_HOLDER_2.clientEntity.toBuilder(); + EntityProto.Builder mutatedTargetEntity2 = REFERENCE_HOLDER_2.targetEntity.toBuilder(); + addProperty(mutatedClientEntity2, "newprop", 999); + addProperty(mutatedTargetEntity2, "newprop", 999); + + // One Entity in the request won't have an id yet. + EntityProto.Builder clientEntityWithNoKey6 = REFERENCE_HOLDER_6.clientEntity.toBuilder(); + EntityProto.Builder targetEntityWithNoKey6 = REFERENCE_HOLDER_6.targetEntity.toBuilder(); + clientEntityWithNoKey6 + .getKeyBuilder() + .getPathBuilder() + .getElementBuilder(0) + .clearName() + .clearId(); + targetEntityWithNoKey6 + .getKeyBuilder() + .getPathBuilder() + .getElementBuilder(0) + .clearName() + .clearId(); + + DatastoreV3Pb.PutRequest putRequest = + DatastoreV3Pb.PutRequest.newBuilder() + .setTransaction(transaction) + .addEntity(mutatedClientEntity2) + .addEntity(clientEntityWithNoKey6) + .build(); + + // It will need to send a Remote Rpc to get an id for this entity. + Reference allocatedKey = expectAllocateIds(targetEntityWithNoKey6.build()).get(0); + + // Returns the target version of the key for the Entity that already had it, and the newly + // allocated key for the other. + DatastoreV3Pb.PutResponse actualPutResp = invokePut(putRequest); + assertThat(actualPutResp.getKeyList()) + .containsExactly(mutatedTargetEntity2.getKey(), allocatedKey) + .inOrder(); + + // Issue another get request. It should use the cached results from the first. It should not + // reflect the change from the Put. + DatastoreV3Pb.GetRequest.Builder getRequest2 = DatastoreV3Pb.GetRequest.newBuilder(); + getRequest2.setTransaction(transaction); + getRequest2.addKey(REFERENCE_HOLDER_1.clientReference); + getRequest2.addKey(REFERENCE_HOLDER_2.clientReference); + getRequest2.addKey(REFERENCE_HOLDER_3.clientReference); + getRequest2.addKey(REFERENCE_HOLDER_4.targetReference); // This key already has the target app. + getRequest2.addKey(REFERENCE_HOLDER_5.clientReference); + + // It doesn't send the request for 1 and 2, because they're already in the cache. + DatastoreV3Pb.GetRequest rewrittenGetRequest2 = + DatastoreV3Pb.GetRequest.newBuilder() + .addKey(REFERENCE_HOLDER_3.targetReference) + .addKey(REFERENCE_HOLDER_4.targetReference) + .addKey(REFERENCE_HOLDER_5.targetReference) + .build(); + + // A mix of Entities, Missing, and Deferred. + DatastoreV3Pb.GetResponse.Builder remoteRpcGetResponse2 = + DatastoreV3Pb.GetResponse.newBuilder(); + remoteRpcGetResponse2.addEntityBuilder().setEntity(REFERENCE_HOLDER_3.targetEntity); + remoteRpcGetResponse2.addDeferred(REFERENCE_HOLDER_4.targetReference); + remoteRpcGetResponse2.addEntityBuilder().setKey(REFERENCE_HOLDER_5.targetReference); + + // Merges both cached data and data from the second GetResponse. + DatastoreV3Pb.GetResponse.Builder expectedGetResponse2 = + DatastoreV3Pb.GetResponse.newBuilder() + .setInOrder(false); // Because there is a deferred result. + expectedGetResponse2.addEntityBuilder().setKey(REFERENCE_HOLDER_1.targetReference); + expectedGetResponse2 + .addEntityBuilder() + .setEntity(REFERENCE_HOLDER_2.targetEntity); // Not from Put. + expectedGetResponse2.addEntityBuilder().setEntity(REFERENCE_HOLDER_3.targetEntity); + expectedGetResponse2.addDeferred(REFERENCE_HOLDER_4.targetReference); + expectedGetResponse2.addEntityBuilder().setKey(REFERENCE_HOLDER_5.targetReference); + + expectRemoteRpcGet(rewrittenGetRequest2, remoteRpcGetResponse2.build()); + + DatastoreV3Pb.GetResponse actualGetResp2 = invokeGet(getRequest2.build()); + assertThat(actualGetResp2).ignoringRepeatedFieldOrder().isEqualTo(expectedGetResponse2.build()); + } + + @Test + public void testQueriesRewriteReferences() { + OnestoreEntity.PropertyValue.ReferenceValue targetRefValue = + OnestoreEntity.PropertyValue.ReferenceValue.newBuilder().setApp(TARGET_APP_ID).build(); + + DatastoreV3Pb.Query.Builder query = DatastoreV3Pb.Query.newBuilder().setApp(TARGET_APP_ID); + query + .addFilterBuilder() + .addPropertyBuilder() + .setName("name") + .setMultiple(false) + .setValue(OnestoreEntity.PropertyValue.newBuilder().setReferenceValue(targetRefValue)); + assertFalse(RemoteDatastore.rewriteQueryAppIds(query, TARGET_APP_ID)); + OnestoreEntity.PropertyValue.ReferenceValue clientRefValue = + OnestoreEntity.PropertyValue.ReferenceValue.newBuilder().setApp(CLIENT_APP_ID).build(); + query = DatastoreV3Pb.Query.newBuilder(); + query.setApp(CLIENT_APP_ID); + query + .addFilterBuilder() + .addPropertyBuilder() + .setName("name") + .setMultiple(false) + .setValue(OnestoreEntity.PropertyValue.newBuilder().setReferenceValue(clientRefValue)); + + // check that a non-reference property is ignored + query + .addFilterBuilder() + .addPropertyBuilder() + .setName("name") + .setMultiple(false) + .setValue( + OnestoreEntity.PropertyValue.newBuilder() + .setStringValue(ByteString.copyFromUtf8("A string"))); + + assertTrue(RemoteDatastore.rewriteQueryAppIds(query, TARGET_APP_ID)); + + assertEquals(TARGET_APP_ID, query.getApp()); + assertEquals( + TARGET_APP_ID, query.getFilter(0).getProperty(0).getValue().getReferenceValue().getApp()); + assertWithMessage("string shouldn't be a reference value") + .that(query.getFilter(1).getProperty(0).getValue().hasReferenceValue()) + .isFalse(); + } + + @Test + public void rewritesPutAppIds() { + OnestoreEntity.PropertyValue.ReferenceValue targetRefValue = + OnestoreEntity.PropertyValue.ReferenceValue.newBuilder() + .setApp(TARGET_APP_ID) + .buildPartial(); + + DatastoreV3Pb.PutRequest.Builder put = DatastoreV3Pb.PutRequest.newBuilder(); + OnestoreEntity.EntityProto.Builder entity = put.addEntityBuilder(); + entity.getKeyBuilder().setApp(TARGET_APP_ID); + entity + .addPropertyBuilder() + .setName("name") + .setMultiple(false) + .setValue(OnestoreEntity.PropertyValue.newBuilder().setReferenceValue(targetRefValue)); + assertFalse(RemoteDatastore.rewritePutAppIds(put, TARGET_APP_ID)); + + OnestoreEntity.PropertyValue.ReferenceValue clientRefValue = + OnestoreEntity.PropertyValue.ReferenceValue.newBuilder() + .setApp(CLIENT_APP_ID) + .buildPartial(); + entity = put.addEntityBuilder(); + entity.getKeyBuilder().setApp(CLIENT_APP_ID); + entity + .addPropertyBuilder() + .setName("name") + .setMultiple(false) + .setValue(OnestoreEntity.PropertyValue.newBuilder().setReferenceValue(clientRefValue)); + + // check that a non-reference property is ignored + entity + .addPropertyBuilder() + .setName("name") + .setMultiple(false) + .setValue( + OnestoreEntity.PropertyValue.newBuilder() + .setStringValue(ByteString.copyFromUtf8("a string"))); + + assertTrue(RemoteDatastore.rewritePutAppIds(put, TARGET_APP_ID)); + + assertEquals(TARGET_APP_ID, entity.getKey().getApp()); + assertEquals(TARGET_APP_ID, entity.getProperty(0).getValue().getReferenceValue().getApp()); + + assertWithMessage("string shouldn't be a reference value") + .that(entity.getProperty(1).getValue().hasReferenceValue()) + .isFalse(); + } + + @Test + public void rewritesAncestorApp() { + OnestoreEntity.Reference.Builder ancestor = + OnestoreEntity.Reference.newBuilder() + .setApp(CLIENT_APP_ID) + .setPath( + OnestoreEntity.Path.newBuilder() + .addElement( + OnestoreEntity.Path.Element.newBuilder().setType("type").setName("name"))); + DatastoreV3Pb.Query.Builder query = + DatastoreV3Pb.Query.newBuilder().setApp(TARGET_APP_ID).setAncestor(ancestor); + assertTrue(RemoteDatastore.rewriteQueryAppIds(query, TARGET_APP_ID)); + ancestor.setApp(TARGET_APP_ID); + query.setAncestor(ancestor); + assertFalse(RemoteDatastore.rewriteQueryAppIds(query, TARGET_APP_ID)); + } + + private static EntityProto createEntityProto(String appId, int index) { + OnestoreEntity.Reference.Builder key = OnestoreEntity.Reference.newBuilder().setApp(appId); + key.getPathBuilder() + .addElement( + OnestoreEntity.Path.Element.newBuilder().setType("somekind").setName("name " + index)); + + EntityProto.Builder entity = EntityProto.newBuilder().setKey(key); + + // TODO: There are utilities for this, but they are all under + // apphosting/datastore/testing which we may not want to depend on from here. + OnestoreEntity.Path group = + OnestoreEntity.Path.newBuilder().addElement(key.getPath().getElement(0)).build(); + entity.setEntityGroup(group); + + addProperty(entity, "someproperty", index); + + return entity.buildPartial(); + } + + private static void addProperty(EntityProto.Builder entity, String propertyName, int value) { + OnestoreEntity.Property property = + OnestoreEntity.Property.newBuilder() + .setName(propertyName) + .setMultiple(false) + .setValue(OnestoreEntity.PropertyValue.newBuilder().setInt64Value(value)) + .build(); + entity.addProperty(property); + } + + /** + * Sets mock expectations for allocating new ids for the given Entities. The RemoteApi + * accomplishes this by calling a special method: remote_datastore.GetIDs. Note that it does not + * actually call allocateIds. + * + * @param entities the entities that need ids allocated. + * @return the References that are mocked to be returned. + */ + private List expectAllocateIds(EntityProto... entities) { + DatastoreV3Pb.PutRequest.Builder expectedReq = DatastoreV3Pb.PutRequest.newBuilder(); + DatastoreV3Pb.PutResponse.Builder resp = DatastoreV3Pb.PutResponse.newBuilder(); + + List allocatedKeys = Lists.newLinkedList(); + int idSeed = 444; + for (OnestoreEntity.EntityProto entity : entities) { + // This is copied from the impl. It sends over empty entities. The key should have a kind, + // but no id/name yet. + OnestoreEntity.EntityProto.Builder reqEntity = expectedReq.addEntityBuilder(); + reqEntity.getKeyBuilder().mergeFrom(entity.getKey()); + reqEntity.getEntityGroupBuilder(); + + // The response will have an id. + Reference.Builder respKey = reqEntity.getKeyBuilder().clone(); + respKey.getPathBuilder().getElementBuilder(0).setId(idSeed); + resp.addKey(respKey); + allocatedKeys.add(respKey.build()); + + idSeed++; + } + + when(mockRemoteRpc.call( + eq(RemoteDatastore.REMOTE_API_SERVICE), + eq("GetIDs"), + eq(""), + eq(expectedReq.build().toByteArray()))) + .thenReturn(resp.build().toByteArray()); + + return allocatedKeys; + } + + private DatastoreV3Pb.Transaction beginTransaction() throws InvalidProtocolBufferException { + DatastoreV3Pb.BeginTransactionRequest beginTxnRequest = + DatastoreV3Pb.BeginTransactionRequest.newBuilder().setApp(CLIENT_APP_ID).build(); + + byte[] txBytes = + remoteDatastore.handleDatastoreCall("BeginTransaction", beginTxnRequest.toByteArray()); + return DatastoreV3Pb.Transaction.parseFrom(txBytes, ExtensionRegistry.getEmptyRegistry()); + } + + private DatastoreV3Pb.GetResponse invokeGet(DatastoreV3Pb.GetRequest req) + throws InvalidProtocolBufferException { + byte[] actualByteResponse = remoteDatastore.handleDatastoreCall("Get", req.toByteArray()); + + return DatastoreV3Pb.GetResponse.parseFrom( + actualByteResponse, ExtensionRegistry.getEmptyRegistry()); + } + + private DatastoreV3Pb.PutResponse invokePut(DatastoreV3Pb.PutRequest req) + throws InvalidProtocolBufferException { + byte[] actualByteResponse = remoteDatastore.handleDatastoreCall("Put", req.toByteArray()); + + return DatastoreV3Pb.PutResponse.parseFrom( + actualByteResponse, ExtensionRegistry.getEmptyRegistry()); + } + + private void expectRemoteRpcGet( + DatastoreV3Pb.GetRequest expectedReq, DatastoreV3Pb.GetResponse resp) { + when(mockRemoteRpc.call( + eq(RemoteDatastore.DATASTORE_SERVICE), + eq("Get"), + eq(""), + eq(expectedReq.toByteArray()))) + .thenReturn(resp.toByteArray()); + } +} diff --git a/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/RemoteRpcTest.java b/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/RemoteRpcTest.java new file mode 100644 index 000000000..9f5cece7f --- /dev/null +++ b/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/RemoteRpcTest.java @@ -0,0 +1,105 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.tools.remoteapi; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth.assertWithMessage; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.google.apphosting.base.protos.api_bytes.RemoteApiPb; +import com.google.protobuf.ExtensionRegistry; +import com.google.protobuf.InvalidProtocolBufferException; +import java.io.IOException; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +/** + * Test for {@link RemoteRpc}. + * + */ +@RunWith(JUnit4.class) +public class RemoteRpcTest { + + @Rule public final MockitoRule mocks = MockitoJUnit.rule(); + + @Mock private AppEngineClient appEngineClient; + + /** + * Test that every RPC has a request id and that these ids differ. + */ + @Test + public void testRequestId() throws IOException { + RemoteRpc remoteRpc = new RemoteRpc(appEngineClient); + byte[] dummyRequest = {1, 2, 3, 4}; + RemoteApiPb.Response responseProto = RemoteApiPb.Response.getDefaultInstance(); + AppEngineClient.Response dummyResponse = + new AppEngineClient.Response(200, responseProto.toByteArray(), UTF_8); + when(appEngineClient.getRemoteApiPath()).thenReturn("/path/one"); + when(appEngineClient + .post(eq("/path/one"), eq("application/octet-stream"), any(byte[].class))) + .thenReturn(dummyResponse); + + ArgumentCaptor byteArrayCaptor = ArgumentCaptor.forClass(byte[].class); + + remoteRpc.call("foo", "bar", "logSuffix", dummyRequest); + + verify(appEngineClient) + .post(eq("/path/one"), eq("application/octet-stream"), byteArrayCaptor.capture()); + RemoteApiPb.Request request1 = parseRequest(byteArrayCaptor.getValue()); + assertThat(request1.getServiceName()).isEqualTo("foo"); + assertThat(request1.getMethod()).isEqualTo("bar"); + assertThat(request1.getRequest().toByteArray()).isEqualTo(dummyRequest); + String id1 = request1.getRequestId(); + + when(appEngineClient.getRemoteApiPath()).thenReturn("/path/two"); + when(appEngineClient + .post(eq("/path/two"), eq("application/octet-stream"), any(byte[].class))) + .thenReturn(dummyResponse); + + remoteRpc.call("foo", "bar", "logSuffix", dummyRequest); + + verify(appEngineClient) + .post(eq("/path/two"), eq("application/octet-stream"), byteArrayCaptor.capture()); + RemoteApiPb.Request request2 = parseRequest(byteArrayCaptor.getValue()); + assertThat(request2.getServiceName()).isEqualTo("foo"); + assertThat(request2.getMethod()).isEqualTo("bar"); + assertThat(request2.getRequest().toByteArray()).isEqualTo(dummyRequest); + String id2 = request2.getRequestId(); + assertWithMessage("Expected '%s' != '%s[", id1, id2).that(id1.equals(id2)).isFalse(); + } + + private static RemoteApiPb.Request parseRequest(byte[] bytes) { + RemoteApiPb.Request.Builder parsedRequest = RemoteApiPb.Request.newBuilder(); + try { + parsedRequest.mergeFrom(bytes, ExtensionRegistry.getEmptyRegistry()); + } catch (InvalidProtocolBufferException e) { + throw new RuntimeException(e); + } + // assertThat(parsedRequest.mergeFrom(bytes)).isTrue(); + return parsedRequest.build(); + } +} diff --git a/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/ThreadLocalDelegateTest.java b/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/ThreadLocalDelegateTest.java new file mode 100644 index 000000000..7b3967cbc --- /dev/null +++ b/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/ThreadLocalDelegateTest.java @@ -0,0 +1,72 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.tools.remoteapi; + +import static com.google.common.truth.Truth.assertThat; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.google.apphosting.api.ApiProxy; +import com.google.apphosting.api.ApiProxy.Delegate; +import com.google.apphosting.api.ApiProxy.Environment; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeoutException; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +/** + */ +@RunWith(JUnit4.class) +public class ThreadLocalDelegateTest { + + @Rule public final MockitoRule mocks = MockitoJUnit.rule(); + + @Mock private Delegate global; + @Mock private Delegate local; + + @Test + public void testSetDelegateForThread() + throws ExecutionException, TimeoutException, InterruptedException { + when(local.getRequestThreads(null)).thenReturn(ImmutableList.of()); + when(global.getRequestThreads(null)).thenReturn(Lists.newArrayList(null, null)); + + ApiProxy.setDelegate(new ThreadLocalDelegate<>(global, local)); + + Delegate delegate = ApiProxy.getDelegate(); + + assertThat(delegate.getRequestThreads(null)).isEmpty(); + + Executors.newSingleThreadExecutor() + .submit(() -> { + Delegate delegate1 = ApiProxy.getDelegate(); + assertThat(delegate1.getRequestThreads(null)).hasSize(2); + }) + .get(1, SECONDS); + + verify(local).getRequestThreads(null); + verify(global).getRequestThreads(null); + } +} diff --git a/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/testing/RemoteApiSharedTests.java b/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/testing/RemoteApiSharedTests.java new file mode 100644 index 000000000..90d7c1c6e --- /dev/null +++ b/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/testing/RemoteApiSharedTests.java @@ -0,0 +1,652 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.tools.remoteapi.testing; + +import com.google.appengine.api.datastore.DatastoreService; +import com.google.appengine.api.datastore.DatastoreServiceFactory; +import com.google.appengine.api.datastore.Entity; +import com.google.appengine.api.datastore.EntityNotFoundException; +import com.google.appengine.api.datastore.Key; +import com.google.appengine.api.datastore.KeyFactory; +import com.google.appengine.api.datastore.Query; +import com.google.appengine.api.datastore.Transaction; +import com.google.appengine.api.datastore.TransactionOptions; +import com.google.appengine.tools.remoteapi.RemoteApiInstaller; +import com.google.appengine.tools.remoteapi.RemoteApiOptions; +import com.google.common.base.Supplier; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.LinkedListMultimap; +import com.google.common.collect.Multimap; +import com.google.errorprone.annotations.CanIgnoreReturnValue; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.logging.Logger; + +/** + * Carries out a series of tests to exercise the Remote API. This can be invoked from within an App + * Engine environment or outside of one. It can target a Java or Python app. + * + */ +public class RemoteApiSharedTests { + + private static final Logger logger = Logger.getLogger(RemoteApiSharedTests.class.getName()); + + private final String localAppId; + private final String remoteAppId; + private final String username; + private final String password; + private final String server; + private final String remoteApiPath; + private final int port; + private final boolean testKeysCreatedBeforeRemoteApiInstall; + private final boolean expectRemoteAppIdsOnKeysAfterInstallingRemoteApi; + + /** + * Builder for a {@link RemoteApiSharedTests} with some sensible defaults. + */ + public static class Builder { + private String localAppId; + private String remoteAppId; + private String username; + private String password; + private String server; + + /** This is the default for Java. Needs to be /_ah/remote_api for Python. */ + private String remoteApiPath = "/remote_api"; + + /** Default for apps running in the App Engine environment. */ + private int port = 443; + + /** + * This should be set to false for Non-Hosted clients. They won't have an Environment available + * to generate Keys until the Remote API is installed. + */ + private boolean testKeysCreatedBeforeRemoteApiInstall = true; + + /** + * Allow these tests to be turned off because they rely on recent changes that haven't been + * rolled out everywhere yet. Targeting 1.8.8. + * See http://b/11254141 and http://b/10788115 + * TODO: Remove this. + */ + private boolean expectRemoteAppIdsOnKeysAfterInstallingRemoteApi = true; + + @CanIgnoreReturnValue + public Builder setLocalAppId(String localAppId) { + this.localAppId = localAppId; + return this; + } + + @CanIgnoreReturnValue + public Builder setRemoteAppId(String remoteAppId) { + this.remoteAppId = remoteAppId; + return this; + } + + @CanIgnoreReturnValue + public Builder setUsername(String username) { + this.username = username; + return this; + } + + @CanIgnoreReturnValue + public Builder setPassword(String password) { + this.password = password; + return this; + } + + @CanIgnoreReturnValue + public Builder setServer(String server) { + this.server = server; + return this; + } + + @CanIgnoreReturnValue + public Builder setPort(int port) { + this.port = port; + return this; + } + + public RemoteApiSharedTests build() { + return new RemoteApiSharedTests( + localAppId, + remoteAppId, + username, + password, + server, + remoteApiPath, + port, + testKeysCreatedBeforeRemoteApiInstall, + expectRemoteAppIdsOnKeysAfterInstallingRemoteApi); + } + } + + private RemoteApiSharedTests( + String localAppId, + String remoteAppId, + String username, + String password, + String server, + String remoteApiPath, + int port, + boolean testKeysCreatedBeforeRemoteApiInstall, + boolean expectRemoteAppIdsOnKeysAfterInstallingRemoteApi) { + this.localAppId = localAppId; + this.remoteAppId = remoteAppId; + this.username = username; + this.password = password; + this.server = server; + this.remoteApiPath = remoteApiPath; + this.port = port; + this.testKeysCreatedBeforeRemoteApiInstall = testKeysCreatedBeforeRemoteApiInstall; + this.expectRemoteAppIdsOnKeysAfterInstallingRemoteApi = + expectRemoteAppIdsOnKeysAfterInstallingRemoteApi; + } + + /** + * Throws an exception if any errors are encountered. If it completes successfully, then all test + * cases passed. + */ + public void runTests() throws IOException { + RemoteApiOptions options = new RemoteApiOptions() + .server(server, port) + .credentials(username, password) + .remoteApiPath(remoteApiPath); + + // Once we install the RemoteApi, all keys will start using the remote app id. We'll store some + // keys with the local app id first. + LocalKeysHolder localKeysHolder = null; + LocalEntitiesHolder localEntitiesHolder = null; + if (testKeysCreatedBeforeRemoteApiInstall) { + localKeysHolder = new LocalKeysHolder(); + localEntitiesHolder = new LocalEntitiesHolder(); + } + + RemoteApiInstaller installer = new RemoteApiInstaller(); + installer.install(options); + + // Update the options with reusable credentials. + options.reuseCredentials(username, installer.serializeCredentials()); + // Execute our tests using the initial installation. + try { + doTest(localKeysHolder, localEntitiesHolder); + } finally { + installer.uninstall(); + } + + if (testKeysCreatedBeforeRemoteApiInstall) { + // Make sure uninstalling brings the keys back to the local app id. + assertNewKeysUseLocalAppId(); + } + + installer.install(options); + // Execute our tests using the second installation. + try { + doTest(localKeysHolder, localEntitiesHolder); + } finally { + installer.uninstall(); + } + } + + /** + * Runs a series of tests using keys with both local app ids and remote app ids. + */ + private void doTest(LocalKeysHolder localKeysHolder, LocalEntitiesHolder localEntitiesHolder) { + DatastoreService ds = DatastoreServiceFactory.getDatastoreService(); + + List tests = ImmutableList.of( + new PutAndGetTester(), + new PutAndGetInTransactionTester(), + new QueryTester(), + new DeleteTester(), + new XgTransactionTester()); + + // Run each test once with local keys and once with remote keys. + for (RemoteApiUnitTest test : tests) { + if (localKeysHolder != null) { + test.run( + ds, + localKeysHolder.createSupplierForFreshKind(), + localEntitiesHolder.createSupplierForFreshKind()); + logger.info("Test passed with local keys: " + test.getClass().getName()); + } + + test.run(ds, new RemoteKeySupplier(), new RemoteEntitySupplier()); + logger.info("Test passed with remote keys: " + test.getClass().getName()); + } + } + + private class PutAndGetTester implements RemoteApiUnitTest { + + @Override + public void run( + DatastoreService ds, Supplier keySupplier, Supplier entitySupplier) { + Entity entity1 = new Entity(keySupplier.get()); + entity1.setProperty("prop1", 75L); + + // Verify results of Put + Key keyReturnedFromPut = ds.put(entity1); + assertEquals(entity1.getKey(), keyReturnedFromPut); + + // Make sure we can retrieve it again. + assertGetEquals(ds, keyReturnedFromPut, entity1); + + // Test EntityNotFoundException + Key unsavedKey = keySupplier.get(); + assertEntityNotFoundException(ds, unsavedKey); + + // Test batch get + Entity entity2 = new Entity(keySupplier.get()); + entity2.setProperty("prop1", 88L); + ds.put(entity2); + + Map batchGetResult = + ds.get(Arrays.asList(entity1.getKey(), unsavedKey, entity2.getKey())); + + // Omits the unsaved key from results. + assertEquals(2, batchGetResult.size()); + assertEquals(entity1, batchGetResult.get(entity1.getKey())); + assertEquals(entity2, batchGetResult.get(entity2.getKey())); + + // Test Put and Get with id generated by Datastore backend. + Entity entity3 = entitySupplier.get(); + entity3.setProperty("prop1", 35L); + assertNoIdOrName(entity3.getKey()); + + Key assignedKey = ds.put(entity3); + + assertTrue(assignedKey.getId() > 0); + assertEquals(assignedKey, entity3.getKey()); + assertGetEquals(ds, assignedKey, entity3); + } + } + + private class PutAndGetInTransactionTester implements RemoteApiUnitTest { + + @Override + public void run( + DatastoreService ds, Supplier keySupplier, Supplier entitySupplier) { + // Put a fresh entity. + Entity originalEntity = new Entity(getFreshKindName()); + originalEntity.setProperty("prop1", 75L); + ds.put(originalEntity); + Key key = originalEntity.getKey(); + + // Prepare a new version of it with a different property value. + Entity mutatedEntity = new Entity(key); + mutatedEntity.setProperty("prop1", 76L); + + // Test Get/Put within a transaction. + Transaction txn = ds.beginTransaction(); + assertGetEquals(ds, key, originalEntity); + ds.put(mutatedEntity); // Write the mutated Entity. + assertGetEquals(ds, key, originalEntity); // Within a txn, the put is not yet reflected. + txn.commit(); + + // Now that the txn is committed, the mutated entity will show up in Get. + assertGetEquals(ds, key, mutatedEntity); + } + } + + private class QueryTester implements RemoteApiUnitTest { + + @Override + public void run( + DatastoreService ds, Supplier keySupplier, Supplier entitySupplier) { + // Note that we can't use local keys here. Query will fail if you set an ancestor whose app + // id does not match the "global" AppIdNamespace. + // TODO: Consider making it more lenient, but it's not a big deal. Users can + // just use a Key that was created after installing the Remote API. + Entity entity = new Entity(getFreshKindName()); + entity.setProperty("prop1", 99L); + ds.put(entity); + + // Make sure we can retrieve it via a query. + Query query = new Query(entity.getKind()); + query.setAncestor(entity.getKey()); + query.setFilter( + new Query.FilterPredicate( + Entity.KEY_RESERVED_PROPERTY, + Query.FilterOperator.GREATER_THAN_OR_EQUAL, + entity.getKey())); + + Entity queryResult = ds.prepare(query).asSingleEntity(); + + // Queries return the Entities with the remote app id. + assertRemoteAppId(queryResult.getKey()); + assertEquals(99L, queryResult.getProperty("prop1")); + } + } + + private class DeleteTester implements RemoteApiUnitTest { + + @Override + public void run( + DatastoreService ds, Supplier keySupplier, Supplier entitySupplier) { + Key key = keySupplier.get(); + Entity entity = new Entity(key); + entity.setProperty("prop1", 75L); + ds.put(entity); + + assertGetEquals(ds, key, entity); + ds.delete(key); + assertEntityNotFoundException(ds, key); + } + } + + private static class XgTransactionTester implements RemoteApiUnitTest { + + @Override + public void run( + DatastoreService ds, Supplier keySupplier, Supplier entitySupplier) { + Transaction txn = ds.beginTransaction(TransactionOptions.Builder.withXG(true)); + if (ds.put(new Entity("xgfoo")).getId() == 0) { + throw new RuntimeException("first entity should have received an id"); + } + if (ds.put(new Entity("xgfoo")).getId() == 0) { + throw new RuntimeException("second entity should have received an id"); + } + txn.commit(); + } + } + + /** + * Simple interface for test cases. + */ + private interface RemoteApiUnitTest { + + /** + * Runs the test case using the given DatastoreService and Keys from the given KeySupplier. + * + * @throws RuntimeException if the test fails. + */ + void run(DatastoreService ds, Supplier keySupplier, Supplier entitySupplier); + } + + /** + * A {@link Supplier} that creates Keys with the Remote app id. Assumes the Remote API is already + * installed. + */ + private class RemoteKeySupplier implements Supplier { + + private final String kind; + private int nameCounter = 0; + + private RemoteKeySupplier() { + this.kind = getFreshKindName(); + } + + @Override + public Key get() { + // This assumes that the remote api has already been installed. + Key key = KeyFactory.createKey(kind, "somename" + nameCounter); + if (expectRemoteAppIdsOnKeysAfterInstallingRemoteApi) { + assertRemoteAppId(key); + } + nameCounter++; + + return key; + } + } + + /** + * A {@link Supplier} that creates Entities with a Key with the Remote app id (and no id or name + * set.) Assumes the Remote API is already installed. + */ + private class RemoteEntitySupplier implements Supplier { + + private final String kind; + + private RemoteEntitySupplier() { + this.kind = getFreshKindName(); + } + + @Override + public Entity get() { + // This assumes that the remote api has already been installed. + Entity entity = new Entity(kind); + if (expectRemoteAppIdsOnKeysAfterInstallingRemoteApi) { + assertRemoteAppId(entity.getKey()); + } + + return entity; + } + } + + /** + * Creates and caches Keys with the local app id. Assumes that the Remote API has not yet been + * installed when a new instance is created. + */ + private class LocalKeysHolder { + private static final int NUM_KINDS = 50; + private static final int NUM_KEYS_PER_KIND = 20; + + private Multimap keysByKind; + + public LocalKeysHolder() { + keysByKind = LinkedListMultimap.create(); + + for (int kindCounter = 0; kindCounter < NUM_KINDS; ++kindCounter) { + String kind = getFreshKindName(); + for (int keyNameCounter = 0; keyNameCounter < NUM_KEYS_PER_KIND; ++keyNameCounter) { + String name = "somename" + keyNameCounter; + + Key key = KeyFactory.createKey(kind, name); + assertLocalAppId(key); + keysByKind.put(kind, key); + } + } + } + + private Supplier createSupplierForFreshKind() { + String kind = keysByKind.keySet().iterator().next(); + final Iterator keysIterator = keysByKind.get(kind).iterator(); + keysByKind.removeAll(kind); + + return new Supplier() { + @Override + public Key get() { + return keysIterator.next(); + } + }; + } + } + + /** + * Creates and caches Entities with Keys containing the local app id (but no id or name set.) + * Assumes that the Remote API has not yet been installed when a new instance is created. + */ + private class LocalEntitiesHolder { + private static final int NUM_KINDS = 50; + private static final int NUM_ENTITIES_PER_KIND = 20; + + private Multimap entitiesByKind; + + public LocalEntitiesHolder() { + entitiesByKind = LinkedListMultimap.create(); + + for (int kindCounter = 0; kindCounter < NUM_KINDS; ++kindCounter) { + String kind = getFreshKindName(); + for (int i = 0; i < NUM_ENTITIES_PER_KIND; ++i) { + // Will get a default Key with the local app id and no id or name. + Entity entity = new Entity(kind); + + assertLocalAppId(entity.getKey()); + entitiesByKind.put(kind, entity); + } + } + } + + private Supplier createSupplierForFreshKind() { + String kind = entitiesByKind.keySet().iterator().next(); + final Iterator entitiesIterator = entitiesByKind.get(kind).iterator(); + entitiesByKind.removeAll(kind); + + return new Supplier() { + @Override + public Entity get() { + return entitiesIterator.next(); + } + }; + } + } + + private void assertTrue(boolean condition, String message) { + if (!condition) { + throw new RuntimeException(message); + } + } + + private void assertTrue(boolean condition) { + assertTrue(condition, ""); + } + + private void assertEquals(Object o1, Object o2) { + assertEquals(o1, o2, "Expected " + o1 + " to equal " + o2); + } + + private void assertEquals(Object o1, Object o2, String message) { + if (o1 == null) { + assertTrue(o2 == null, message); + return; + } + assertTrue(o1.equals(o2), message); + } + + /** Special version of assertEquals for Entities that will ignore app ids on Keys. */ + private void assertEquals(Entity e1, Entity e2) { + if (e1 == null) { + assertTrue(e2 == null); + return; + } + + assertEquals(e1.getProperties(), e2.getProperties()); + assertEquals(e1.getKey(), e2.getKey()); + } + + /** Special version of assertEquals for Keys that will ignore app ids. */ + private void assertEquals(Key k1, Key k2) { + if (k1 == null) { + assertTrue(k2 == null); + return; + } + + assertEquals(k1.getKind(), k2.getKind()); + assertEquals(k1.getId(), k2.getId()); + assertEquals(k1.getName(), k2.getName()); + + assertEquals(k1.getParent(), k2.getParent()); + } + + private void assertLocalAppId(Key key) { + assertAppIdsMatchIgnoringPartition(localAppId, key.getAppId()); + } + + private void assertRemoteAppId(Key key) { + assertAppIdsMatchIgnoringPartition(remoteAppId, key.getAppId()); + } + + /** + * The e2e testing framework is not very strict about requiring fully specified app ids. + * Therefore, we might get "display" app ids given to us and we need to consider "s~foo" and "foo" + * to be equal. + */ + private void assertAppIdsMatchIgnoringPartition(String appId1, String appId2) { + if (appId1.equals(appId2)) { + // Exact match. + return; + } + + // Consider s~foo == foo. + assertEquals( + stripPartitionFromAppId(appId1), + stripPartitionFromAppId(appId2), + "Expected app id to be: " + appId1 + ", but was: " + appId2); + } + + /** + * Example conversions: + * + * foo => foo + * s~foo => foo + * e~foo => foo + * hrd~foo => foo (Doesn't exist in App Engine today, but this code will support partitions + * greater than 1 char.) + */ + private String stripPartitionFromAppId(String appId) { + int partitionIndex = appId.indexOf('~'); + if (partitionIndex != -1 && appId.length() > partitionIndex + 1) { + return appId.substring(partitionIndex + 1); + } + return appId; + } + + private void assertNewKeysUseLocalAppId() { + assertLocalAppId(KeyFactory.createKey(getFreshKindName(), "somename")); + } + + private void assertNoIdOrName(Key key) { + assertEquals(0L, key.getId()); + assertEquals(null, key.getName()); + } + + private void assertGetEquals(DatastoreService ds, Key keyToGet, Entity expectedEntity) { + // Test the single key api. + Entity entityFromGet = quietGet(ds, keyToGet); + assertEquals(expectedEntity, entityFromGet); + assertRemoteAppId(entityFromGet.getKey()); + + // Test the multi-get api. + Map getResults = ds.get(Collections.singletonList(keyToGet)); + assertEquals(1, getResults.size()); + Entity entityFromBatchGet = getResults.get(keyToGet); + assertEquals(expectedEntity, entityFromBatchGet); + assertRemoteAppId(entityFromBatchGet.getKey()); + } + + private void assertEntityNotFoundException(DatastoreService ds, Key missingKey) { + try { + ds.get(missingKey); + throw new RuntimeException("Did not receive expected exception"); + } catch (EntityNotFoundException e) { + // expected + } + } + + /** + * Propagates {@link EntityNotFoundException} as {@link RuntimeException} + */ + private Entity quietGet(DatastoreService ds, Key key) { + try { + return ds.get(key); + } catch (EntityNotFoundException e) { + throw new RuntimeException(e); + } + } + + /** + * Generates a Kind name that has not yet been used. + */ + private String getFreshKindName() { + return "testkind" + UUID.randomUUID(); + } +} diff --git a/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/testing/StubCredential.java b/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/testing/StubCredential.java new file mode 100644 index 000000000..b40d35ef3 --- /dev/null +++ b/remoteapi/src/test/java/com/google/appengine/tools/remoteapi/testing/StubCredential.java @@ -0,0 +1,40 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.tools.remoteapi.testing; + +import com.google.api.client.auth.oauth2.Credential; +import com.google.api.client.http.HttpRequest; +import java.io.IOException; + +/** + * A stub {@link Credential} for testing. + */ +public class StubCredential extends Credential { + + public StubCredential() { + super(new AccessMethod() { + @Override + public String getAccessTokenFromRequest(HttpRequest request) { + return null; + } + + @Override + public void intercept(HttpRequest request, String accessToken) throws IOException { + } + }); + } +} diff --git a/remoteapi/src/test/resources/com/google/appengine/tools/remoteapi/testdata/test.pkcs12 b/remoteapi/src/test/resources/com/google/appengine/tools/remoteapi/testdata/test.pkcs12 new file mode 100644 index 000000000..51c07e509 Binary files /dev/null and b/remoteapi/src/test/resources/com/google/appengine/tools/remoteapi/testdata/test.pkcs12 differ diff --git a/runtime/annotationscanningwebapp/pom.xml b/runtime/annotationscanningwebapp/pom.xml index a224717e5..d39fa1cb3 100644 --- a/runtime/annotationscanningwebapp/pom.xml +++ b/runtime/annotationscanningwebapp/pom.xml @@ -66,7 +66,7 @@ maven-compiler-plugin - 3.14.1 + 3.15.0 8 diff --git a/runtime/annotationscanningwebappjakarta/pom.xml b/runtime/annotationscanningwebappjakarta/pom.xml index f7a4003d5..0c40d62b6 100644 --- a/runtime/annotationscanningwebappjakarta/pom.xml +++ b/runtime/annotationscanningwebappjakarta/pom.xml @@ -66,7 +66,7 @@ maven-compiler-plugin - 3.14.1 + 3.15.0 8 diff --git a/runtime/failinitfilterwebapp/pom.xml b/runtime/failinitfilterwebapp/pom.xml index f308b73d3..9e5eec31f 100644 --- a/runtime/failinitfilterwebapp/pom.xml +++ b/runtime/failinitfilterwebapp/pom.xml @@ -66,7 +66,7 @@ maven-compiler-plugin - 3.14.1 + 3.15.0 8 diff --git a/runtime/failinitfilterwebappjakarta/pom.xml b/runtime/failinitfilterwebappjakarta/pom.xml index cfed42784..378b3f511 100644 --- a/runtime/failinitfilterwebappjakarta/pom.xml +++ b/runtime/failinitfilterwebappjakarta/pom.xml @@ -66,7 +66,7 @@ maven-compiler-plugin - 3.14.1 + 3.15.0 8 diff --git a/runtime/impl/src/main/java/com/google/apphosting/runtime/ApiDeadlineMap.java b/runtime/impl/src/main/java/com/google/apphosting/runtime/ApiDeadlineMap.java new file mode 100644 index 000000000..2bfc06115 --- /dev/null +++ b/runtime/impl/src/main/java/com/google/apphosting/runtime/ApiDeadlineMap.java @@ -0,0 +1,123 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.apphosting.runtime; + +import com.google.common.collect.ImmutableMap; + +/** + * Default deadline maps for API calls, distinguishing between online and offline requests. + * + *

In App Engine, requests are treated as either 'online' or 'offline', with different request + * time limits and API call deadline limits applied to each. + * + *

An online request is a standard HTTP request from an end-user and must typically complete + * within 60 seconds. + * + *

An offline request is one that is not directly serving an incoming, user-initiated HTTP + * request and is allowed to run for a longer duration (e.g., 10 minutes or more). This typically + * includes background tasks such as: + * + *

    + *
  • Task Queue tasks (from push or pull queues) + *
  • Cron jobs + *
+ * + *

Because background tasks often perform long-running operations like data processing or batch + * updates, they require longer deadlines. This class provides separate deadline maps for API calls + * made during online vs. offline requests. For example, API calls like URL Fetch ({@code urlfetch}) + * or Cloud SQL ({@code rdbms}) have a much higher maximum deadline (e.g., 600 seconds) in the + * offline maps, allowing background tasks to complete long-running API calls without timing out. + */ +public class ApiDeadlineMap { + + /** Map of package name to default API call deadline in seconds for online requests. */ + public static final ImmutableMap DEFAULT_DEADLINE_MAP = + ImmutableMap.builder() + .put("app_config_service", 60.0) + .put("blobstore", 15.0) + .put("datastore_v3", 60.0) + .put("datastore_v4", 60.0) + .put("file", 30.0) + .put("images", 30.0) + .put("logservice", 60.0) + .put("mail", 30.0) + .put("modules", 60.0) + .put("rdbms", 60.0) + .put("remote_socket", 60.0) + .put("search", 10.0) + .put("stubby", 10.0) + .buildOrThrow(); + + /** Map of package name to maximum API call deadline in seconds for online requests. */ + public static final ImmutableMap MAX_DEADLINE_MAP = + ImmutableMap.builder() + .put("app_config_service", 60.0) + .put("blobstore", 30.0) + .put("datastore_v3", 270.0) + .put("datastore_v4", 270.0) + .put("file", 60.0) + .put("images", 30.0) + .put("logservice", 60.0) + .put("mail", 60.0) + .put("modules", 60.0) + .put("rdbms", 60.0) + .put("remote_socket", 60.0) + .put("search", 60.0) + .put("stubby", 60.0) + .put("taskqueue", 30.0) + .put("urlfetch", 60.0) + .buildOrThrow(); + + /** Map of package name to default API call deadline in seconds for offline requests. */ + public static final ImmutableMap OFFLINE_DEFAULT_DEADLINE_MAP = + ImmutableMap.builder() + .put("app_config_service", 60.0) + .put("blobstore", 15.0) + .put("datastore_v3", 60.0) + .put("datastore_v4", 60.0) + .put("file", 30.0) + .put("images", 30.0) + .put("logservice", 60.0) + .put("mail", 30.0) + .put("modules", 60.0) + .put("rdbms", 60.0) + .put("remote_socket", 60.0) + .put("search", 10.0) + .put("stubby", 10.0) + .buildOrThrow(); + + /** Map of package name to maximum API call deadline in seconds for offline requests. */ + public static final ImmutableMap OFFLINE_MAX_DEADLINE_MAP = + ImmutableMap.builder() + .put("app_config_service", 60.0) + .put("blobstore", 30.0) + .put("datastore_v3", 270.0) + .put("datastore_v4", 270.0) + .put("file", 60.0) + .put("images", 30.0) + .put("logservice", 60.0) + .put("mail", 60.0) + .put("modules", 60.0) + .put("rdbms", 600.0) + .put("remote_socket", 60.0) + .put("search", 60.0) + .put("stubby", 600.0) + .put("taskqueue", 30.0) + .put("urlfetch", 600.0) + .buildOrThrow(); + + private ApiDeadlineMap() {} +} diff --git a/runtime/impl/src/main/java/com/google/apphosting/runtime/ApiDeadlineOracle.java b/runtime/impl/src/main/java/com/google/apphosting/runtime/ApiDeadlineOracle.java index ce0a705d6..f36b8e1b8 100644 --- a/runtime/impl/src/main/java/com/google/apphosting/runtime/ApiDeadlineOracle.java +++ b/runtime/impl/src/main/java/com/google/apphosting/runtime/ApiDeadlineOracle.java @@ -16,36 +16,41 @@ package com.google.apphosting.runtime; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.util.HashMap; import java.util.Map; /** - * {@code ApiDeadlineOracle} determines the appropriate deadline for - * API calls based on the user-specified deadline, the per-package - * maximum and default deadlines, and the fallthrough maximum and - * default deadlines. - * - *

This class is also used to track shared buffer counts and sizes - * as they can also be specified on a per-package and online/offline - * basis. + * {@code ApiDeadlineOracle} determines the appropriate deadline for API calls based on the + * user-specified deadline, the per-package maximum and default deadlines, and the fallthrough + * maximum and default deadlines. * + *

This class is also used to track shared buffer counts and sizes as they can also be specified + * on a per-package and online/offline basis. */ public class ApiDeadlineOracle { private final DeadlineMap deadlineMap; - private final DeadlineMap offlineDeadlineMap; - // TODO: Rename this class to something less deadline-specific. - private ApiDeadlineOracle(DeadlineMap deadlineMap, DeadlineMap offlineDeadlineMap) { + private ApiDeadlineOracle(DeadlineMap deadlineMap) { this.deadlineMap = deadlineMap; - this.offlineDeadlineMap = offlineDeadlineMap; } public double getDeadline(String packageName, boolean isOffline, Number userDeadline) { if (isOffline) { - return offlineDeadlineMap.getDeadline(packageName, userDeadline); - } else { - return deadlineMap.getDeadline(packageName, userDeadline); + double deadline; + if (userDeadline == null) { + deadline = + ApiDeadlineMap.OFFLINE_DEFAULT_DEADLINE_MAP.getOrDefault( + packageName, 5.0); // Default offline deadline + } else { + deadline = userDeadline.doubleValue(); + } + return Math.min( + deadline, + ApiDeadlineMap.OFFLINE_MAX_DEADLINE_MAP.getOrDefault( + packageName, 10.0)); // Default offline max deadline } + return deadlineMap.getDeadline(packageName, userDeadline); } public void addPackageDefaultDeadline(String packageName, double defaultDeadline) { @@ -56,14 +61,6 @@ public void addPackageMaxDeadline(String packageName, double maxDeadline) { deadlineMap.addMaxDeadline(packageName, maxDeadline); } - public void addOfflinePackageDefaultDeadline(String packageName, double defaultDeadline) { - offlineDeadlineMap.addDefaultDeadline(packageName, defaultDeadline); - } - - public void addOfflinePackageMaxDeadline(String packageName, double maxDeadline) { - offlineDeadlineMap.addMaxDeadline(packageName, maxDeadline); - } - public void addPackageMinContentSizeForBuffer(String packageName, long minContentSizeForBuffer) { deadlineMap.addMinContentSizeForBuffer(packageName, minContentSizeForBuffer); } @@ -72,29 +69,16 @@ public void addPackageMaxRequestSize(String packageName, long maxRequestSize) { deadlineMap.addMaxRequestSize(packageName, maxRequestSize); } - public void addOfflinePackageMinContentSizeForBuffer( - String packageName, long minContentSizeForBuffer) { - offlineDeadlineMap.addMinContentSizeForBuffer(packageName, minContentSizeForBuffer); - } - - public void addOfflinePackageMaxRequestSize(String packageName, long maxRequestSize) { - offlineDeadlineMap.addMaxRequestSize(packageName, maxRequestSize); - } - /** Build an ApiDeadlineOracle. */ public static class Builder { private DeadlineMap deadlineMap; - private DeadlineMap offlineDeadlineMap; - public Builder initDeadlineMap( - double defaultDeadline, - String defaultDeadlineMapString, - double maxDeadline, - String maxDeadlineMapString) { + /** Initializes the default deadline map using standard hardcoded values. */ + @CanIgnoreReturnValue + public Builder initDeadlineMap() { deadlineMap = new DeadlineMap( - defaultDeadline, parseDoubleMap(defaultDeadlineMapString), - maxDeadline, parseDoubleMap(maxDeadlineMapString)); + 10.0, ApiDeadlineMap.DEFAULT_DEADLINE_MAP, 10.0, ApiDeadlineMap.MAX_DEADLINE_MAP); return this; } @@ -103,44 +87,11 @@ public Builder initDeadlineMap(DeadlineMap deadlineMap) { return this; } - public Builder initOfflineDeadlineMap( - double defaultDeadline, - String defaultDeadlineMapString, - double maxDeadline, - String maxDeadlineMapString) { - offlineDeadlineMap = - new DeadlineMap( - defaultDeadline, - parseDoubleMap(defaultDeadlineMapString), - maxDeadline, - parseDoubleMap(maxDeadlineMapString)); - return this; - } - - public Builder initOfflineDeadlineMap(DeadlineMap offlineDeadlineMap) { - this.offlineDeadlineMap = offlineDeadlineMap; - return this; - } - public ApiDeadlineOracle build() { - if (deadlineMap == null || offlineDeadlineMap == null) { + if (deadlineMap == null) { throw new IllegalStateException("All deadline maps must be initialized."); } - return new ApiDeadlineOracle(deadlineMap, offlineDeadlineMap); - } - - private static Map parseDoubleMap(String mapString) { - Map map = new HashMap(); - if (mapString.length() > 0) { - for (String entry : mapString.split(",")) { - int colon = entry.indexOf(':'); - if (colon == -1) { - throw new IllegalArgumentException("Could not parse entry: " + entry); - } - map.put(entry.substring(0, colon), Double.parseDouble(entry.substring(colon + 1))); - } - } - return map; + return new ApiDeadlineOracle(deadlineMap); } } @@ -159,9 +110,9 @@ public DeadlineMap( double maxDeadline, Map maxDeadlineMap) { this.defaultDeadline = defaultDeadline; - this.defaultDeadlineMap = defaultDeadlineMap; + this.defaultDeadlineMap = new HashMap<>(defaultDeadlineMap); this.maxDeadline = maxDeadline; - this.maxDeadlineMap = maxDeadlineMap; + this.maxDeadlineMap = new HashMap<>(maxDeadlineMap); this.minContentSizeForBufferMap = new HashMap(); this.maxRequestSizeMap = new HashMap(); } diff --git a/runtime/impl/src/main/java/com/google/apphosting/runtime/ApiProxyImpl.java b/runtime/impl/src/main/java/com/google/apphosting/runtime/ApiProxyImpl.java index 33ea176b8..3af70c613 100644 --- a/runtime/impl/src/main/java/com/google/apphosting/runtime/ApiProxyImpl.java +++ b/runtime/impl/src/main/java/com/google/apphosting/runtime/ApiProxyImpl.java @@ -160,7 +160,6 @@ public class ApiProxyImpl implements ApiProxy.Delegate future) { private static Map createInitialAttributes( RequestAPIData request, String externalDatacenterName, - BackgroundRequestCoordinator coordinator, - boolean cloudSqlJdbcConnectivityEnabled) { + BackgroundRequestCoordinator coordinator) { Map attributes = new HashMap<>(); attributes.put(USER_ID_KEY, request.getObfuscatedGaiaId()); attributes.put(USER_ORGANIZATION_KEY, request.getUserOrganization()); @@ -1131,7 +1119,7 @@ private static Map createInitialAttributes( attributes.put(REQUEST_THREAD_FACTORY_ATTR, CurrentRequestThreadFactory.SINGLETON); attributes.put(BACKGROUND_THREAD_FACTORY_ATTR, new BackgroundThreadFactory(coordinator)); - attributes.put(CLOUD_SQL_JDBC_CONNECTIVITY_ENABLED_KEY, cloudSqlJdbcConnectivityEnabled); + attributes.put(CLOUD_SQL_JDBC_CONNECTIVITY_ENABLED_KEY, true); // Environments are associated with requests, and can now be // shared across more than one thread. We'll synchronize all diff --git a/runtime/impl/src/main/java/com/google/apphosting/runtime/AppEngineConstants.java b/runtime/impl/src/main/java/com/google/apphosting/runtime/AppEngineConstants.java index 308cb9d98..7665f6647 100644 --- a/runtime/impl/src/main/java/com/google/apphosting/runtime/AppEngineConstants.java +++ b/runtime/impl/src/main/java/com/google/apphosting/runtime/AppEngineConstants.java @@ -132,5 +132,25 @@ public final class AppEngineConstants { public static final String IGNORE_RESPONSE_SIZE_LIMIT = "appengine.ignore.responseSizeLimit"; + /** + * If positive, send thread interrupts this many milliseconds before the hard deadline. + */ + public static final int SOFT_DEADLINE_DELAY_MS = 10600; + + /** + * How many CPU cycles should be assumed per second for billing purposes. + */ + public static final long CYCLES_PER_SECOND = 1000000000L; + + /** + * The Jetty request header size in bytes (256K). + */ + public static final int JETTY_REQUEST_HEADER_SIZE = 262144; + + /** + * The Jetty response header size in bytes (256K). + */ + public static final int JETTY_RESPONSE_HEADER_SIZE = 262144; + private AppEngineConstants() {} } diff --git a/runtime/runtime_impl_jetty9/src/main/java/com/google/apphosting/runtime/jetty9/AppInfoFactory.java b/runtime/impl/src/main/java/com/google/apphosting/runtime/AppInfoFactory.java similarity index 87% rename from runtime/runtime_impl_jetty9/src/main/java/com/google/apphosting/runtime/jetty9/AppInfoFactory.java rename to runtime/impl/src/main/java/com/google/apphosting/runtime/AppInfoFactory.java index 3f6c16283..71cb2532b 100644 --- a/runtime/runtime_impl_jetty9/src/main/java/com/google/apphosting/runtime/jetty9/AppInfoFactory.java +++ b/runtime/impl/src/main/java/com/google/apphosting/runtime/AppInfoFactory.java @@ -14,10 +14,10 @@ * limitations under the License. */ -package com.google.apphosting.runtime.jetty9; +package com.google.apphosting.runtime; + import com.google.apphosting.base.protos.AppinfoPb; -import com.google.apphosting.utils.config.AppYaml; import java.io.File; import java.io.IOException; import java.nio.file.NoSuchFileException; @@ -84,17 +84,9 @@ public AppinfoPb.AppInfo getAppInfoFromFile(String applicationRoot, String fixed return getAppInfo(); } - public AppinfoPb.AppInfo getAppInfoFromAppYaml(AppYaml unused) throws IOException { - return getAppInfo(); - } - public AppinfoPb.AppInfo getAppInfo() { - final AppinfoPb.AppInfo.Builder appInfoBuilder = - AppinfoPb.AppInfo.newBuilder() - .setAppId(gaeApplication) - .setVersionId(gaeVersion) - .setRuntimeId("java8"); - return appInfoBuilder.build(); + public AppinfoPb.AppInfo getAppInfo() { + return AppinfoPb.AppInfo.newBuilder().setAppId(gaeApplication).setVersionId(gaeVersion).build(); } } diff --git a/runtime/impl/src/main/java/com/google/apphosting/runtime/AppVersionFactory.java b/runtime/impl/src/main/java/com/google/apphosting/runtime/AppVersionFactory.java index 7df67ffc6..c06e9eaf1 100644 --- a/runtime/impl/src/main/java/com/google/apphosting/runtime/AppVersionFactory.java +++ b/runtime/impl/src/main/java/com/google/apphosting/runtime/AppVersionFactory.java @@ -72,10 +72,7 @@ public static Builder builder() { public static Builder builderForTest() { return builder() - .setDefaultToNativeUrlStreamHandler(true) .setForceUrlfetchUrlStreamHandler(false) - .setIgnoreDaemonThreads(true) - .setUseEnvVarsFromAppInfo(false) .setFixedApplicationPath(null); } @@ -90,14 +87,8 @@ public abstract static class Builder { /** The runtime version which is reported to users. */ public abstract Builder setRuntimeVersion(String x); - public abstract Builder setDefaultToNativeUrlStreamHandler(boolean x); - public abstract Builder setForceUrlfetchUrlStreamHandler(boolean x); - public abstract Builder setIgnoreDaemonThreads(boolean x); - - public abstract Builder setUseEnvVarsFromAppInfo(boolean x); - public abstract Builder setFixedApplicationPath(String x); public abstract AppVersionFactory build(); @@ -113,14 +104,8 @@ public abstract static class Builder { private final String runtimeVersion; - private final boolean defaultToNativeUrlStreamHandler; - private final boolean forceUrlfetchUrlStreamHandler; - private final boolean ignoreDaemonThreads; - - private final boolean useEnvVarsFromAppInfo; - private final String fixedApplicationPath; /** Construct a new {@code AppVersionFactory}. */ @@ -128,18 +113,12 @@ public AppVersionFactory( NullSandboxPlugin sandboxPlugin, File sharedDirectory, String runtimeVersion, - boolean defaultToNativeUrlStreamHandler, boolean forceUrlfetchUrlStreamHandler, - boolean ignoreDaemonThreads, - boolean useEnvVarsFromAppInfo, @Nullable String fixedApplicationPath) { this.sandboxPlugin = sandboxPlugin; this.sharedDirectory = sharedDirectory; this.runtimeVersion = runtimeVersion; - this.defaultToNativeUrlStreamHandler = defaultToNativeUrlStreamHandler; this.forceUrlfetchUrlStreamHandler = forceUrlfetchUrlStreamHandler; - this.ignoreDaemonThreads = ignoreDaemonThreads; - this.useEnvVarsFromAppInfo = useEnvVarsFromAppInfo; this.fixedApplicationPath = fixedApplicationPath; } @@ -216,7 +195,7 @@ public AppVersion createAppVersion( configuration); String urlStreamHandlerType = appEngineWebXml.getUrlStreamHandlerType(); - if (urlStreamHandlerType == null && defaultToNativeUrlStreamHandler) { + if (urlStreamHandlerType == null) { urlStreamHandlerType = AppEngineWebXml.URL_HANDLER_NATIVE; } if (forceUrlfetchUrlStreamHandler) { @@ -253,7 +232,7 @@ public AppVersion createAppVersion( .setParentThreadGroup(rootThreadGroup) .setThreadGroupNamePrefix("Request #") .setUncaughtExceptionHandler(uncaughtExceptionHandler) - .setIgnoreDaemonThreads(ignoreDaemonThreads) + .setIgnoreDaemonThreads(true) .build(); setApplicationDirectory(rootDirectory.getAbsolutePath()); return AppVersion.builder() @@ -344,14 +323,12 @@ private Map createEnvironmentVariables(AppEngineWebXml appEngine AppInfo appInfo) { Map envVars = new HashMap<>(); envVars.putAll(appEngineWebXml.getEnvironmentVariables()); - if (useEnvVarsFromAppInfo) { - // We add env vars from AppInfo on top of those from appengine-web.xml because - // for a long time Java appcfg was not correctly populating the env_variables - // section in the generated app.yaml (see b/79371098), meaning they were missing - // from AppInfos of deployed apps. - for (AppInfo.EnvironmentVariable envVar : appInfo.getEnvironmentVariableList()) { - envVars.put(envVar.getName(), envVar.getValue()); - } + // We add env vars from AppInfo on top of those from appengine-web.xml because + // for a long time Java appcfg was not correctly populating the env_variables + // section in the generated app.yaml (see b/79371098), meaning they were missing + // from AppInfos of deployed apps. + for (AppInfo.EnvironmentVariable envVar : appInfo.getEnvironmentVariableList()) { + envVars.put(envVar.getName(), envVar.getValue()); } String gaeEnv = System.getenv(GAE_ENV); if (USE_DEFAULT_VALUES_FOR_GAE_ENV_VARS && gaeEnv == null) { diff --git a/runtime/impl/src/main/java/com/google/apphosting/runtime/CloneControllerImpl.java b/runtime/impl/src/main/java/com/google/apphosting/runtime/CloneControllerImpl.java deleted file mode 100644 index 7f4ad6782..000000000 --- a/runtime/impl/src/main/java/com/google/apphosting/runtime/CloneControllerImpl.java +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.apphosting.runtime; - -import com.google.apphosting.base.protos.ClonePb.ApiPackageDeadlines; -import com.google.apphosting.base.protos.ClonePb.CloneSettings; -import com.google.apphosting.base.protos.ClonePb.PerformanceData; -import com.google.apphosting.base.protos.EmptyMessage; -import com.google.apphosting.base.protos.ModelClonePb.DeadlineInfo; -import com.google.apphosting.base.protos.ModelClonePb.PerformanceDataRequest; -import com.google.apphosting.runtime.anyrpc.AnyRpcServerContext; -import com.google.apphosting.runtime.anyrpc.CloneControllerServerInterface; -import com.google.common.flogger.GoogleLogger; -import com.google.protobuf.ByteString; -import java.nio.ByteBuffer; - -/** - * {@code CloneControllerImpl} implements the {@link CloneControllerServerInterface} RPC interface. - */ -public class CloneControllerImpl implements CloneControllerServerInterface { - private static final GoogleLogger logger = GoogleLogger.forEnclosingClass(); - - private final Callback callback; - private final ApiDeadlineOracle deadlineOracle; - private final RequestManager requestManager; - private final ByteBuffer hotspotPerformanceData; - - public CloneControllerImpl( - Callback callback, - ApiDeadlineOracle deadlineOracle, - RequestManager requestManager, - ByteBuffer hotspotPerformanceData) { - this.callback = callback; - this.deadlineOracle = deadlineOracle; - this.requestManager = requestManager; - this.hotspotPerformanceData = hotspotPerformanceData; - } - - /** - * Obsolete operation. This was used by an earlier sandboxing scheme, now obsolete. - */ - @Override - public void waitForSandbox(AnyRpcServerContext rpc, EmptyMessage unused) { - rpc.finishWithAppError(1, "waitForSandbox is unimplemented"); - } - - /** - * Applies the specified {@link CloneSettings} received from the - * AppServer. These settings cannot be known at clone start-up - * because they may vary by application. - */ - @Override - public void applyCloneSettings(AnyRpcServerContext rpc, CloneSettings settings) { - logger.atWarning().log("applyCloneSettings"); - try { - // Historically we translated the max_virtual_memory_mb and max_cpu_seconds fields from the - // CloneSettings into setrlimit calls here. But it no longer makes sense to depend on the - // runtime to impose these limits. Instead, the containing sandbox should do it. - - if (settings.hasMaxOutstandingApiRpcs()) { - // This must be less than --clone_max_outstanding_api_rpcs - // because we specify that value when creating the Stubby - // channel. - requestManager.setMaxOutstandingApiRpcs(settings.getMaxOutstandingApiRpcs()); - } - - // Now apply any package deadline overrides that we received. - for (ApiPackageDeadlines deadline : settings.getApiCallDeadlinesList()) { - if (deadline.hasDefaultDeadlineS()) { - deadlineOracle.addPackageDefaultDeadline( - deadline.getApiPackage(), deadline.getDefaultDeadlineS()); - } - if (deadline.hasMaxDeadlineS()) { - deadlineOracle.addPackageMaxDeadline( - deadline.getApiPackage(), deadline.getMaxDeadlineS()); - } - } - for (ApiPackageDeadlines deadline : settings.getOfflineApiCallDeadlinesList()) { - if (deadline.hasDefaultDeadlineS()) { - deadlineOracle.addOfflinePackageDefaultDeadline( - deadline.getApiPackage(), deadline.getDefaultDeadlineS()); - } - if (deadline.hasMaxDeadlineS()) { - deadlineOracle.addOfflinePackageMaxDeadline( - deadline.getApiPackage(), deadline.getMaxDeadlineS()); - } - } - - // Switch network services to use the App Engine Socket API. - callback.divertNetworkServices(); - rpc.finishWithResponse(EmptyMessage.getDefaultInstance()); - logger.atWarning().log("applyCloneSettings done"); - } catch (RuntimeException ex) { - logger.atSevere().withCause(ex).log("oh noes"); - throw ex; - } - } - - @Override - public void sendDeadline(final AnyRpcServerContext rpc, DeadlineInfo deadline) { - logger.atInfo().log("Got a sendDeadline RPC."); - requestManager.sendDeadline(deadline.getSecurityTicket(), deadline.getHard()); - rpc.finishWithResponse(EmptyMessage.getDefaultInstance()); - } - - @Override - public void getPerformanceData(AnyRpcServerContext rpc, PerformanceDataRequest req) { - logger.atInfo().log("Got a getPerformanceData RPC with type %s", req.getType()); - PerformanceData.Builder data = PerformanceData.newBuilder().setType(req.getType()); - if (hotspotPerformanceData != null) { - PerformanceData.Entry.Builder entry = - PerformanceData.Entry.newBuilder() - .setFormat(PerformanceData.Format.JAVA_HOTSPOT_HSPERFDATA); - ByteBuffer bb = hotspotPerformanceData.duplicate(); - bb.position(0); - bb.limit(bb.capacity()); - entry.setPayload(ByteString.copyFrom(bb)); - data.addEntries(entry); - } - rpc.finishWithResponse(data.build()); - } - - /** - * Callback interface for rpc-specific and sandbox-specific functionality to be abstracted - * over in this class. - */ - public interface Callback { - /** - * Start re-routing the socket API in the JRE through the GAE socket API. - */ - void divertNetworkServices(); - - AppVersion getAppVersion(String appId, String versionId); - } -} diff --git a/runtime/impl/src/main/java/com/google/apphosting/runtime/JavaRuntime.java b/runtime/impl/src/main/java/com/google/apphosting/runtime/JavaRuntime.java index 9785ccfe9..ab5b00cce 100644 --- a/runtime/impl/src/main/java/com/google/apphosting/runtime/JavaRuntime.java +++ b/runtime/impl/src/main/java/com/google/apphosting/runtime/JavaRuntime.java @@ -16,35 +16,21 @@ package com.google.apphosting.runtime; -import static com.google.common.base.StandardSystemProperty.JAVA_SPECIFICATION_VERSION; - import com.google.apphosting.base.AppVersionKey; import com.google.apphosting.base.protos.AppinfoPb.AppInfo; import com.google.apphosting.base.protos.EmptyMessage; import com.google.apphosting.base.protos.RuntimePb.UPAddDelete; import com.google.apphosting.base.protos.RuntimePb.UPRequest; import com.google.apphosting.base.protos.RuntimePb.UPResponse; -import com.google.apphosting.runtime.anyrpc.AnyRpcPlugin; import com.google.apphosting.runtime.anyrpc.AnyRpcServerContext; import com.google.apphosting.runtime.anyrpc.EvaluationRuntimeServerInterface; import com.google.apphosting.utils.config.AppEngineWebXml; import com.google.auto.value.AutoBuilder; -import com.google.common.annotations.VisibleForTesting; import com.google.common.flogger.GoogleLogger; import java.io.ByteArrayInputStream; import java.io.File; -import java.lang.reflect.Field; -import java.lang.reflect.Method; -import java.lang.reflect.Modifier; -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.nio.ByteBuffer; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.SynchronousQueue; import org.jspecify.annotations.Nullable; /** @@ -62,9 +48,6 @@ public class JavaRuntime implements EvaluationRuntimeServerInterface { private static final GoogleLogger logger = GoogleLogger.forEnclosingClass(); - /** Environment Variable for logging messages to /var/log. */ - private static final String VAR_LOG_ENV_VAR = "WRITE_LOGS_TO_VAR_LOG"; - /** Environment variable for the GCP project. */ private static final String GOOGLE_CLOUD_PROJECT_ENV_VAR = "GOOGLE_CLOUD_PROJECT"; @@ -83,9 +66,6 @@ public class JavaRuntime implements EvaluationRuntimeServerInterface { /** Sandbox-agnostic plugin. */ private final NullSandboxPlugin sandboxPlugin; - /** RPC-agnostic plugin. */ - private final AnyRpcPlugin rpcPlugin; - /** {@code AppVersionFactory} can construct {@link AppVersion} instances. */ private final AppVersionFactory appVersionFactory; @@ -98,34 +78,14 @@ public class JavaRuntime implements EvaluationRuntimeServerInterface { /** A template runtime configuration for applications. */ private final ApplicationEnvironment.RuntimeConfiguration templateConfiguration; - /** The object responsible for choosing API call deadlines. */ - private final ApiDeadlineOracle deadlineOracle; - private final Logging logging = new Logging(); private final BackgroundRequestCoordinator coordinator; - private final boolean compressResponse; - - private final boolean enableHotspotPerformanceMetrics; - - private final boolean pollForNetwork; - - private final boolean redirectStdoutStderr; - - private final boolean logJsonToFile; - private final boolean clearLogHandlers; private final Path jsonLogDir; - /** - * This will contain a reference to the ByteBuffer containing Hotspot performance data, exported - * by the sun.misc.Perf api in Java 8 and by jdk.internal.perf.Perf in Java 9. It's set once and - * for all when start() is called. - */ - private ByteBuffer hotspotPerformanceData = null; - /** * The app version that has been received by this runtime, or null if no version has been received * yet. We only ever receive one version. @@ -135,16 +95,8 @@ public class JavaRuntime implements EvaluationRuntimeServerInterface { /** Get a partly-initialized builder. */ public static Builder builder() { return new AutoBuilder_JavaRuntime_Builder() - .setCompressResponse(true) - .setEnableHotspotPerformanceMetrics(true) - .setPollForNetwork(false) - .setDefaultToNativeUrlStreamHandler(false) .setForceUrlfetchUrlStreamHandler(false) - .setIgnoreDaemonThreads(true) - .setUseEnvVarsFromAppInfo(false) .setFixedApplicationPath(null) - .setRedirectStdoutStderr(true) - .setLogJsonToFile(false) .setClearLogHandlers(true) .setJsonLogDir(DEFAULT_JSON_LOG_OUTPUT_DIR); } @@ -160,10 +112,6 @@ public abstract static class Builder { public abstract Builder setSandboxPlugin(NullSandboxPlugin sandboxPlugin); - public abstract Builder setRpcPlugin(AnyRpcPlugin rpcPlugin); - - public abstract AnyRpcPlugin rpcPlugin(); - public abstract Builder setSharedDirectory(File sharedDirectory); public abstract File sharedDirectory(); @@ -185,48 +133,14 @@ public abstract Builder setConfiguration( public abstract Builder setCoordinator(BackgroundRequestCoordinator coordinator); - public abstract Builder setCompressResponse(boolean compressResponse); - - public abstract boolean compressResponse(); - - public abstract Builder setEnableHotspotPerformanceMetrics( - boolean enableHotspotPerformanceMetrics); - - public abstract boolean enableHotspotPerformanceMetrics(); - - public abstract Builder setPollForNetwork(boolean pollForNetwork); - - public abstract boolean pollForNetwork(); - - public abstract Builder setDefaultToNativeUrlStreamHandler( - boolean defaultToNativeUrlStreamHandler); - - public abstract boolean defaultToNativeUrlStreamHandler(); - public abstract Builder setForceUrlfetchUrlStreamHandler(boolean forceUrlfetchUrlStreamHandler); public abstract boolean forceUrlfetchUrlStreamHandler(); - public abstract Builder setIgnoreDaemonThreads(boolean ignoreDaemonThreads); - - public abstract boolean ignoreDaemonThreads(); - - public abstract Builder setUseEnvVarsFromAppInfo(boolean useEnvVarsFromAppInfo); - - public abstract boolean useEnvVarsFromAppInfo(); - public abstract Builder setFixedApplicationPath(String fixedApplicationPath); public abstract String fixedApplicationPath(); - public abstract Builder setRedirectStdoutStderr(boolean redirect); - - public abstract boolean redirectStdoutStderr(); - - public abstract Builder setLogJsonToFile(boolean log); - - public abstract boolean logJsonToFile(); - public abstract Builder setClearLogHandlers(boolean log); public abstract Builder setJsonLogDir(Path path); @@ -239,49 +153,30 @@ public abstract Builder setDefaultToNativeUrlStreamHandler( JavaRuntime( ServletEngineAdapter servletEngine, NullSandboxPlugin sandboxPlugin, - AnyRpcPlugin rpcPlugin, File sharedDirectory, RequestManager requestManager, String runtimeVersion, ApplicationEnvironment.RuntimeConfiguration configuration, ApiDeadlineOracle deadlineOracle, BackgroundRequestCoordinator coordinator, - boolean compressResponse, - boolean enableHotspotPerformanceMetrics, - boolean pollForNetwork, - boolean defaultToNativeUrlStreamHandler, boolean forceUrlfetchUrlStreamHandler, - boolean ignoreDaemonThreads, - boolean useEnvVarsFromAppInfo, @Nullable String fixedApplicationPath, - boolean redirectStdoutStderr, - boolean logJsonToFile, boolean clearLogHandlers, Path jsonLogDir) { this.servletEngine = servletEngine; this.sandboxPlugin = sandboxPlugin; - this.rpcPlugin = rpcPlugin; this.requestManager = requestManager; this.appVersionFactory = AppVersionFactory.builder() .setSandboxPlugin(sandboxPlugin) .setSharedDirectory(sharedDirectory) .setRuntimeVersion(runtimeVersion) - .setDefaultToNativeUrlStreamHandler(defaultToNativeUrlStreamHandler) .setForceUrlfetchUrlStreamHandler(forceUrlfetchUrlStreamHandler) - .setIgnoreDaemonThreads(ignoreDaemonThreads) - .setUseEnvVarsFromAppInfo(useEnvVarsFromAppInfo) .setFixedApplicationPath(fixedApplicationPath) .build(); this.runtimeVersion = runtimeVersion; this.templateConfiguration = configuration; - this.deadlineOracle = deadlineOracle; this.coordinator = coordinator; - this.compressResponse = compressResponse; - this.enableHotspotPerformanceMetrics = enableHotspotPerformanceMetrics; - this.pollForNetwork = pollForNetwork; - this.redirectStdoutStderr = redirectStdoutStderr; - this.logJsonToFile = logJsonToFile; this.clearLogHandlers = clearLogHandlers; this.jsonLogDir = jsonLogDir; } @@ -293,69 +188,12 @@ public abstract Builder setDefaultToNativeUrlStreamHandler( public void start(ServletEngineAdapter.Config runtimeOptions) { logger.atInfo().log("JavaRuntime starting..."); - if (enableHotspotPerformanceMetrics) { - try { - // The Perf class is in different packages in Java 8 and Java 9. - try { - hotspotPerformanceData = getPerformanceDataByteBuffer("sun.misc.Perf"); - } catch (ClassNotFoundException e) { - hotspotPerformanceData = getPerformanceDataByteBuffer("jdk.internal.perf.Perf"); - } - } catch (Exception e) { - logger.atWarning().withCause(e).log("Failed to access Hotspot performance data"); - } - } - - SynchronousQueue rpcStarted = new SynchronousQueue<>(); - - new Thread(new RpcRunnable(rpcStarted), "Runtime Network Thread").start(); - // Wait for the servlet engine to start up. servletEngine.start("Google App Engine/" + runtimeVersion, runtimeOptions); - - // Wait for our rpc service to start up. - Object response; - try { - response = rpcStarted.take(); - } catch (InterruptedException ex) { - throw new RuntimeException("Interrupted while starting runtime", ex); - } - if (response instanceof Error) { - throw (Error) response; - } else if (response instanceof RuntimeException) { - throw (RuntimeException) response; - } else if (response instanceof Throwable) { - throw new RuntimeException(((Throwable) response)); - } else if (response instanceof AnyRpcPlugin) { - // Success. Ignore the result. When it comes time to stop the server, - // we'll use the rpcPlugin we have - } else { - throw new RuntimeException("Unknown response: " + response); - } - } - - private ByteBuffer getPerformanceDataByteBuffer(String perfClassName) - throws ReflectiveOperationException { - // Attaching to the current process (lvmid == 0) returns a shared buffer valid - // for the entire life of the JVM. - // The following code is equivalent to: - // ByteBuffer buffer = Perf.getPerf().attach(0, "r"); - // We invoke it reflectively because this class has been renamed in Java 9. - Class perfClass = Class.forName(perfClassName); - Method getPerfMethod = perfClass.getMethod("getPerf"); - Object perf = getPerfMethod.invoke(null); - Method attachMethod = perf.getClass().getMethod("attach", int.class, String.class); - ByteBuffer buffer = (ByteBuffer) attachMethod.invoke(perf, 0, "r"); - if (buffer.capacity() == 0) { - throw new RuntimeException("JVM does not export Hotspot performance data"); - } - return buffer; } /** Perform a graceful shutdown of our RPC service, and then shut down our servlet engine. */ public void stop() { - logger.atInfo().log("JavaRuntime stopping..."); - rpcPlugin.stopServer(); logger.atInfo().log("JavaRuntime stopped."); servletEngine.stop(); } @@ -395,14 +233,13 @@ public void handleRequest(AnyRpcServerContext rpc, UPRequest upRequest) { .setUpResponse(upResponse) .setRequestManager(requestManager) .setCoordinator(coordinator) - .setCompressResponse(compressResponse) .setUpRequestHandler(servletEngine) .build(); appVersion .getThreadGroupPool() .start( "Request" + upRequest.getEventIdHash(), - rpcPlugin.traceContextPropagating(requestRunner)); + requestRunner); } catch (InterruptedException ex) { RequestRunner.setFailure( upResponse, @@ -443,32 +280,19 @@ public synchronized void addAppVersion(AnyRpcServerContext rpc, AppInfo appInfo) ApplicationEnvironment env = appVersion.getEnvironment(); - if ("1.8".equals(JAVA_SPECIFICATION_VERSION.value())) { - setEnvironmentVariables(env.getEnvironmentVariables()); - } System.getProperties().putAll(env.getSystemProperties()); // NOTE: This string should be kept in sync with the one used by the // Logger_.getPrivateContextName(String) method. - String identifier = env.getAppId() + "/" + env.getVersionId(); String userLogConfigFilePath = env.getSystemProperties().get("java.util.logging.config.file"); if (userLogConfigFilePath != null) { userLogConfigFilePath = env.getRootDirectory().getAbsolutePath() + "/" + userLogConfigFilePath; } System.setIn(new ByteArrayInputStream(new byte[0])); - if (logJsonToFile || "1".equals(System.getenv(VAR_LOG_ENV_VAR))) { - logging.logJsonToFile( - System.getenv(GOOGLE_CLOUD_PROJECT_ENV_VAR), - jsonLogDir.resolve(JSON_LOG_OUTPUT_FILE), - clearLogHandlers); - } else { - sandboxPlugin.startCapturingApplicationLogs(); - } - if (redirectStdoutStderr) { - // Reassign the standard streams so that e.g. System.out.println works as intended, - // i.e. it sends output to the application log. - logging.redirectStdoutStderr(identifier); - } + logging.logJsonToFile( + System.getenv(GOOGLE_CLOUD_PROJECT_ENV_VAR), + jsonLogDir.resolve(JSON_LOG_OUTPUT_FILE), + clearLogHandlers); logging.applyLogProperties(userLogConfigFilePath, sandboxPlugin.getApplicationClassLoader()); // Now notify the servlet engine, so it can do any setup it // has to do. @@ -483,15 +307,6 @@ public synchronized void addAppVersion(AnyRpcServerContext rpc, AppInfo appInfo) rpc.finishWithResponse(EmptyMessage.getDefaultInstance()); } - /** - * Obsolete operation. Deleting app versions has always been theoretically possible but never - * actually implemented in App Engine. - */ - @Override - public synchronized void deleteAppVersion(AnyRpcServerContext rpc, AppInfo appInfo) { - rpc.finishWithAppError(UPAddDelete.ERROR.FAILURE_VALUE, "Version deletion is unimplemented"); - } - synchronized AppVersion findAppVersion(String appId, String versionId) { AppVersionKey key = AppVersionKey.of(appId, versionId); if (key.equals(appVersion.getKey())) { @@ -513,128 +328,4 @@ public static void killCloneIfSeriousException(Throwable th) { } } } - - private static void setEnvironmentVariables(Map vars) { - // Setting the environment variables after the JVM has started requires a bit of a hack: - // we reach into the package-private java.lang.ProcessEnvironment class, which incidentally - // is platform-specific, and replace the map held in a static final field there, - // using yet more reflection. - Map allVars = new HashMap<>(System.getenv()); - vars.forEach( - (k, v) -> { - if (v == null) { - logger.atWarning().log("Null value for $%s", k); - } - allVars.put(k, v); - }); - try { - Class pe = Class.forName("java.lang.ProcessEnvironment", true, null); - Field f = pe.getDeclaredField("theUnmodifiableEnvironment"); - f.setAccessible(true); - Field m = Field.class.getDeclaredField("modifiers"); - m.setAccessible(true); - m.setInt(f, m.getInt(f) & ~Modifier.FINAL); - f.set(null, Collections.unmodifiableMap(allVars)); - } catch (ReflectiveOperationException e) { - throw new RuntimeException("failed to set the environment variables", e); - } - } - - private class RpcRunnable implements Runnable { - private final SynchronousQueue rpcStarted; - - RpcRunnable(SynchronousQueue rpcStarted) { - this.rpcStarted = rpcStarted; - } - - @Override - public void run() { - try { - // NOTE: This method never returns -- this thread is now the - // network thread and will be responsible for accepting socket - // connections in a loop and handing off control to the - // Executor created above. - startServer(); - } catch (Throwable ex) { - logger.atSevere().withCause(ex).log("JavaRuntime server could not start"); - try { - // Something went wrong. Pass the exception back. - rpcStarted.put(ex); - } catch (InterruptedException ex2) { - throw new RuntimeException(ex2); - } - } - } - - @SuppressWarnings("SystemExitOutsideMain") - private void startServer() throws Exception { - CloneControllerImplCallback callback = new CloneControllerImplCallback(); - CloneControllerImpl controller = - new CloneControllerImpl( - callback, deadlineOracle, requestManager, hotspotPerformanceData); - rpcPlugin.startServer(JavaRuntime.this, controller); - - rpcStarted.put(rpcPlugin); - - try { - logger.atInfo().log("Beginning accept loop."); - // This must run in the same thread that created the EventDispatcher. - rpcPlugin.blockUntilShutdown(); - } catch (Throwable ex) { - // We've already called rpcStarted.put() so there's no - // sense trying to pass the exception -- no one is waiting any - // longer. Instead, just print what we can and kill the - // server. Without a network thread we cannot send a response - // back to the AppServer anyway. - ex.printStackTrace(); - System.exit(1); - } - } - } - - @VisibleForTesting - class CloneControllerImplCallback implements CloneControllerImpl.Callback { - @Override - public void divertNetworkServices() { - if (pollForNetwork) { - pollNetworkingReady(); - } - } - - @Override - public AppVersion getAppVersion(String appId, String versionId) { - return findAppVersion(appId, versionId); - } - - // Swallow the checked exception that Thread.sleep() has. - private void sleep(int time) { - try { - Thread.sleep(time); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - } - - private void pollNetworkingReady() { - logger.atInfo().log("Polling for if networking is ready."); - long start = System.nanoTime(); - // TODO(b/33757746): The gateway client seems to require multiple seconds to be ready. - for (int i = 0; i < 100; i++) { - try { - InetAddress.getByName("google.com"); - long finish = System.nanoTime(); - // If networking is NOT ready, then getByName should throw an exception. - logger.atInfo().log("Networking ready. Polled for %.3f s.", (finish - start) / 1e9); - return; - } catch (UnknownHostException e) { - // We expect to get exceptions when the gateway client is still - // starting up. - logger.atInfo().withCause(e).log("Couldn't connect"); - sleep(100); - } - } - logger.atSevere().log("Could not verify that networking is ready."); - throw new RuntimeException("Cannot verify that networking is ready"); - } - } } diff --git a/runtime/impl/src/main/java/com/google/apphosting/runtime/JavaRuntimeFactory.java b/runtime/impl/src/main/java/com/google/apphosting/runtime/JavaRuntimeFactory.java index 4fc391ac1..958c89b22 100644 --- a/runtime/impl/src/main/java/com/google/apphosting/runtime/JavaRuntimeFactory.java +++ b/runtime/impl/src/main/java/com/google/apphosting/runtime/JavaRuntimeFactory.java @@ -17,9 +17,7 @@ package com.google.apphosting.runtime; import com.google.apphosting.api.ApiProxy; -import com.google.apphosting.runtime.anyrpc.AnyRpcPlugin; import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.VerifyException; import com.google.common.flogger.GoogleLogger; import com.google.common.net.HostAndPort; import java.io.File; @@ -78,39 +76,20 @@ public JavaRuntime getStartedRuntime(NullSandboxPlugin sandboxPlugin, String[] a System.setProperty("appengine.urlfetch.deriveResponseMessage", "true"); } - if (params.getMailSupportExtendedAttachmentEncodings()) { - // This system property is checked by GMTransport, which is directly - // registered with JavaMail and cannot take additional constructor arguments. - System.setProperty("appengine.mail.supportExtendedAttachmentEncodings", "true"); - } + // This system property is checked by GMTransport, which is directly + // registered with JavaMail and cannot take additional constructor arguments. + System.setProperty("appengine.mail.supportExtendedAttachmentEncodings", "true"); - if (params.getForceReadaheadOnCloudsqlSocket()) { - System.setProperty("appengine.jdbc.forceReadaheadOnCloudsqlSocket", "true"); - } + System.setProperty("appengine.jdbc.forceReadaheadOnCloudsqlSocket", "true"); - if (params.getMailFilenamePreventsInlining()) { - // This system property is checked by GMTransport, which is directly - // registered with JavaMail and cannot take additional constructor arguments. - System.setProperty("appengine.mail.filenamePreventsInlining", "true"); - } + // This system property is checked by GMTransport, which is directly + // registered with JavaMail and cannot take additional constructor arguments. + System.setProperty("appengine.mail.filenamePreventsInlining", "true"); ServletEngineAdapter servletEngine = createServletEngine(params); ApiDeadlineOracle deadlineOracle = - new ApiDeadlineOracle.Builder() - .initDeadlineMap( - params.getApiCallDeadline(), - params.getApiCallDeadlineMap(), - params.getMaxApiCallDeadline(), - params.getMaxApiCallDeadlineMap()) - .initOfflineDeadlineMap( - params.getOfflineApiCallDeadline(), - params.getOfflineApiCallDeadlineMap(), - params.getMaxOfflineApiCallDeadline(), - params.getMaxOfflineApiCallDeadlineMap()) - .build(); + new ApiDeadlineOracle.Builder().initDeadlineMap().build(); - AnyRpcPlugin rpcPlugin = loadRpcPlugin(params); - rpcPlugin.initialize(params.getPort()); ApiHostClientFactory apiHostFactory = new ApiHostClientFactory(); BackgroundRequestCoordinator coordinator = new BackgroundRequestCoordinator(); @@ -122,68 +101,52 @@ public JavaRuntime getStartedRuntime(NullSandboxPlugin sandboxPlugin, String[] a params.getTrustedHost(), OptionalInt.of(params.getCloneMaxOutstandingApiRpcs()))) .setDeadlineOracle(deadlineOracle) - .setExternalDatacenterName(params.getExternalDatacenterName()) + .setExternalDatacenterName("MARS") .setByteCountBeforeFlushing(params.getByteCountBeforeFlushing()) .setMaxLogLineSize(params.getMaxLogLineSize()) .setMaxLogFlushTime(Duration.ofSeconds(params.getMaxLogFlushSeconds())) .setCoordinator(coordinator) - .setCloudSqlJdbcConnectivityEnabled(params.getEnableGaeCloudSqlJdbcConnectivity()) .setDisableApiCallLogging(params.getDisableApiCallLogging()) .build(); RequestManager.Builder requestManagerBuilder = RequestManager.builder() - .setSoftDeadlineDelay(params.getJavaSoftDeadlineMs()) + .setSoftDeadlineDelay(AppEngineConstants.SOFT_DEADLINE_DELAY_MS) .setRuntimeLogSink(Optional.of(logSink)) .setApiProxyImpl(apiProxyImpl) .setMaxOutstandingApiRpcs(params.getCloneMaxOutstandingApiRpcs()) .setThreadStopTerminatesClone(params.getThreadStopTerminatesClone()) - .setInterruptFirstOnSoftDeadline(params.getInterruptThreadsFirstOnSoftDeadline()) - .setCyclesPerSecond(params.getCyclesPerSecond()) - .setWaitForDaemonRequestThreads(params.getWaitForDaemonRequestThreads()); + .setCyclesPerSecond(AppEngineConstants.CYCLES_PER_SECOND); RequestManager requestManager = makeRequestManager(requestManagerBuilder); apiProxyImpl.setRequestManager(requestManager); ApplicationEnvironment.RuntimeConfiguration configuration = - ApplicationEnvironment.RuntimeConfiguration.builder() - .setCloudSqlJdbcConnectivityEnabled(params.getEnableGaeCloudSqlJdbcConnectivity()) - .setUseGoogleConnectorJ(params.getDefaultUseGoogleConnectorj()) - .build(); + ApplicationEnvironment.RuntimeConfiguration.builder().build(); JavaRuntime.Builder runtimeBuilder = JavaRuntime.builder() .setServletEngine(servletEngine) .setSandboxPlugin(sandboxPlugin) - .setRpcPlugin(rpcPlugin) - .setSharedDirectory(new File(params.getApplicationRoot())) + .setSharedDirectory(new File("notused")) .setRequestManager(requestManager) - .setRuntimeVersion("Google App Engine/" + params.getAppengineReleaseName()) + .setRuntimeVersion("Google App Engine/" + "mainwithdefaults") .setConfiguration(configuration) .setDeadlineOracle(deadlineOracle) .setCoordinator(coordinator) - .setCompressResponse(params.getRuntimeHttpCompression()) - .setEnableHotspotPerformanceMetrics(params.getEnableHotspotPerformanceMetrics()) - .setPollForNetwork(params.getPollForNetwork()) - .setDefaultToNativeUrlStreamHandler(params.getDefaultToNativeUrlStreamHandler()) .setForceUrlfetchUrlStreamHandler(params.getForceUrlfetchUrlStreamHandler()) - .setIgnoreDaemonThreads(!params.getWaitForDaemonRequestThreads()) - .setUseEnvVarsFromAppInfo(params.getUseEnvVarsFromAppInfo()) - .setFixedApplicationPath(params.getFixedApplicationPath()) - .setRedirectStdoutStderr(!params.getUseJettyHttpProxy()) - .setLogJsonToFile(params.getLogJsonToVarLog()); + .setFixedApplicationPath(params.getFixedApplicationPath()); JavaRuntime runtime = makeRuntime(runtimeBuilder); ApiProxy.setDelegate(apiProxyImpl); ServletEngineAdapter.Config runtimeOptions = ServletEngineAdapter.Config.builder() - .setUseJettyHttpProxy(params.getUseJettyHttpProxy()) - .setApplicationRoot(params.getApplicationRoot()) + .setApplicationRoot("notused") .setFixedApplicationPath(params.getFixedApplicationPath()) .setJettyHttpAddress(HostAndPort.fromParts("0.0.0.0", params.getJettyHttpPort())) - .setJettyRequestHeaderSize(params.getJettyRequestHeaderSize()) - .setJettyResponseHeaderSize(params.getJettyResponseHeaderSize()) + .setJettyRequestHeaderSize(AppEngineConstants.JETTY_REQUEST_HEADER_SIZE) + .setJettyResponseHeaderSize(AppEngineConstants.JETTY_RESPONSE_HEADER_SIZE) .setEvaluationRuntimeServerInterface(runtime) .build(); try { @@ -207,14 +170,6 @@ public RequestManager makeRequestManager(RequestManager.Builder builder) { return builder.build(); } - private static AnyRpcPlugin loadRpcPlugin(JavaRuntimeParams params) { - if (params.getUseJettyHttpProxy()) { - return new NullRpcPlugin(); - }else { - throw new VerifyException("Sorry, the gen1 GrpcPlugin is not supported anymore."); - } - } - /** Creates the ServletEngineAdapter specifies by the --servlet_engine flag. */ private static ServletEngineAdapter createServletEngine(JavaRuntimeParams params) { Class engineClazz = params.getServletEngine(); diff --git a/runtime/impl/src/main/java/com/google/apphosting/runtime/JavaRuntimeParams.java b/runtime/impl/src/main/java/com/google/apphosting/runtime/JavaRuntimeParams.java index e4d71fc6b..983d60a4a 100644 --- a/runtime/impl/src/main/java/com/google/apphosting/runtime/JavaRuntimeParams.java +++ b/runtime/impl/src/main/java/com/google/apphosting/runtime/JavaRuntimeParams.java @@ -29,72 +29,12 @@ final class JavaRuntimeParams { private Class servletEngineClass; - @Parameter( - description = "Root path for application data on the local filesystem.", - names = {"--application_root"}) - private String applicationRoot = "appdata"; - - @Parameter( - description = "Port number to expose our EvaluationRuntime service on.", - names = {"--port"}) - private int port = 0; - @Parameter( description = "Specification used for connecting back to the appserver.", names = {"--trusted_host"}) private String trustedHost = ""; - @Parameter( - description = - "Number of milliseconds before the deadline for a request " - + "to throw a catchable exception.", - names = {"--java_soft_deadline_ms"}) - private int javaSoftDeadlineMs = 600; - - @Parameter( - description = "Default deadline for all API RPCs, in seconds.", - names = {"--api_call_deadline"}) - private double apiCallDeadline = 5.0; - - @Parameter( - description = "Maximum deadline for all API RPCs, in seconds.", - names = {"--max_api_call_deadline"}) - private double maxApiCallDeadline = 10.0; - - @Parameter( - description = "Default deadline for all API RPCs by package in seconds.", - names = {"--api_call_deadline_map"}) - private String apiCallDeadlineMap = ""; - - @Parameter( - description = "Maximum deadline for all API RPCs by package in seconds.", - names = {"--max_api_call_deadline_map"}) - private String maxApiCallDeadlineMap = ""; - - @Parameter( - description = "Default deadline for all offline API RPCs, in seconds.", - names = {"--offline_api_call_deadline"}) - private double offlineApiCallDeadline = 5.0; - - @Parameter( - description = "Maximum deadline for all offline API RPCs, in seconds.", - names = {"--max_offline_api_call_deadline"}) - private double maxOfflineApiCallDeadline = 10.0; - - @Parameter( - description = "Default deadline for all offline API RPCs by package in seconds.", - names = {"--offline_api_call_deadline_map"}) - private String offlineApiCallDeadlineMap = ""; - - @Parameter( - description = "Maximum deadline for all offline API RPCs by package in seconds.", - names = {"--max_offline_api_call_deadline_map"}) - private String maxOfflineApiCallDeadlineMap = ""; - @Parameter( - description = "The name for the current release of Google App Engine.", - names = {"--appengine_release_name"}) - private String appengineReleaseName = "unknown"; @Parameter( description = "If true, exceptions logged by Jetty also go to app logs.", @@ -102,11 +42,6 @@ final class JavaRuntimeParams { arity = 1) private boolean logJettyExceptionsToAppLogs = true; - @Parameter( - description = "Identifier for this datacenter.", - names = {"--external_datacenter_name"}) - private String externalDatacenterName = null; - @Parameter( description = "The maximum number of simultaneous APIHost RPCs.", names = {"--clone_max_outstanding_api_rpcs"}) @@ -142,12 +77,6 @@ final class JavaRuntimeParams { names = {"--max_log_flush_seconds"}) private int maxLogFlushSeconds = 60; - @Parameter( - description = "Compress HTTP responses in the runtime.", - names = {"--runtime_http_compression"}, - arity = 1) - private boolean runtimeHttpCompression = false; - @Parameter( description = "The maximum allowed size in bytes of the Runtime Log " @@ -155,30 +84,6 @@ final class JavaRuntimeParams { names = {"--max_runtime_log_per_request"}) private long maxRuntimeLogPerRequest = 3000L * 1024L; - @Parameter( - description = - "Whether to use the JDBC connectivity for accessing Cloud SQL " - + "through the AppEngine Java applications.", - names = {"--enable_gae_cloud_sql_jdbc_connectivity"}, - arity = 1) - private boolean enableGaeCloudSqlJdbcConnectivity = false; - - @Parameter( - description = - "Whether to use google connector-j by default even if it's not explicitly set in" - + " appengine-web.xml.", - names = {"--default_use_google_connectorj"}, - arity = 1) - private boolean defaultUseGoogleConnectorj = false; - - @Parameter( - description = - "On a soft deadline, attempt to interrupt application threads first, then " - + "stop them only if necessary", - names = {"--interrupt_threads_first_on_soft_deadline"}, - arity = 1) - private boolean interruptThreadsFirstOnSoftDeadline = false; - @Parameter( description = "Whether to enable exporting of hotspot performance metrics.", names = {"--enable_hotspot_performance_metrics"}, @@ -203,36 +108,7 @@ final class JavaRuntimeParams { arity = 1) private boolean urlfetchDeriveResponseMessage = true; - @Parameter( - description = "Prevent the Mail API from inlining attachments with filenames.", - names = {"--mail_filename_prevents_inlining"}, - arity = 1) - private boolean mailFilenamePreventsInlining = false; - - @Parameter( - description = "Support byte[] and nested Multipart-encoded Mail attachments", - names = {"--mail_support_extended_attachment_encodings"}, - arity = 1) - private boolean mailSupportExtendedAttachmentEncodings = false; - @Parameter( - description = "Always enable readahead on a CloudSQL socket", - names = {"--force_readahead_on_cloudsql_socket"}, - arity = 1) - private boolean forceReadaheadOnCloudsqlSocket = false; - - @Parameter( - description = "Speed of the processor in clock cycles per second.", - names = {"--cycles_per_second"}, - arity = 1) - private long cyclesPerSecond = 0L; - - @Parameter( - description = - "Wait for request threads with the daemon bit set before considering a request complete.", - names = {"--wait_for_daemon_request_threads"}, - arity = 1) - private boolean waitForDaemonRequestThreads = true; @Parameter( description = "Poll for network connectivity before running application code.", @@ -240,12 +116,6 @@ final class JavaRuntimeParams { arity = 1) private boolean pollForNetwork = false; - @Parameter( - description = "Default url-stream-handler to 'native' instead of 'urlfetch'.", - names = {"--default_to_native_url_stream_handler", "--default_to_builtin_url_stream_handler"}, - arity = 1) - private boolean defaultToNativeUrlStreamHandler = false; - @Parameter( description = "Force url-stream-handler to 'urlfetch' irrespective of the contents " @@ -254,20 +124,6 @@ final class JavaRuntimeParams { arity = 1) private boolean forceUrlfetchUrlStreamHandler = false; - @Parameter( - description = "Enable synchronization inside of AppLogsWriter.", - names = {"--enable_synchronized_app_logs_writer"}, - arity = 1) - private boolean enableSynchronizedAppLogsWriter = true; - - @Parameter( - description = - "Use environment variables from the AppInfo instead of those " - + "in the appengine-web.xml descriptor.", - names = {"--use_env_vars_from_app_info"}, - arity = 1) - private boolean useEnvVarsFromAppInfo = false; - @Parameter( description = "Fixed path to use for the application root directory, irrespective of " @@ -275,28 +131,12 @@ final class JavaRuntimeParams { names = {"--fixed_application_path"}) private String fixedApplicationPath = null; - @Parameter( - description = - "Enable a Jetty server listening to HTTP requests and forwarding via RPC to " - + "the java runtime.", - names = {"--use_jetty_http_proxy"}, - arity = 1) - private boolean useJettyHttpProxy = false; - @Parameter( description = "Jetty HTTP Port number to use for http access to the runtime.", names = {"--jetty_http_port"}) private int jettyHttpPort = 8080; - @Parameter( - description = "Jetty server's max size for HTTP request headers.", - names = {"--jetty_request_header_size"}) - private int jettyRequestHeaderSize = 16384; - @Parameter( - description = "Jetty server's max size for HTTP response headers.", - names = {"--jetty_response_header_size"}) - private int jettyResponseHeaderSize = 16384; @Parameter( description = "Disable API call logging in the runtime.", @@ -304,12 +144,6 @@ final class JavaRuntimeParams { arity = 1) private boolean disableApiCallLogging = false; - @Parameter( - description = "Configure java.util.logging to log JSON messages to /var/log/app.", - names = {"--log_json_to_var_log"}, - arity = 1) - private boolean logJsonToVarLog = false; - private List unknownParams; private JavaRuntimeParams() {} @@ -380,61 +214,11 @@ private void initServletEngineClass() { } } - String getApplicationRoot() { - return applicationRoot; - } - - int getPort() { - return port; - } - String getTrustedHost() { return trustedHost; } - int getJavaSoftDeadlineMs() { - return javaSoftDeadlineMs; - } - - double getApiCallDeadline() { - return apiCallDeadline; - } - - double getMaxApiCallDeadline() { - return maxApiCallDeadline; - } - - String getApiCallDeadlineMap() { - return apiCallDeadlineMap; - } - - String getMaxApiCallDeadlineMap() { - return maxApiCallDeadlineMap; - } - - double getOfflineApiCallDeadline() { - return offlineApiCallDeadline; - } - - double getMaxOfflineApiCallDeadline() { - return maxOfflineApiCallDeadline; - } - - String getOfflineApiCallDeadlineMap() { - return offlineApiCallDeadlineMap; - } - - String getMaxOfflineApiCallDeadlineMap() { - return maxOfflineApiCallDeadlineMap; - } - - String getAppengineReleaseName() { - return appengineReleaseName; - } - String getExternalDatacenterName() { - return externalDatacenterName; - } int getCloneMaxOutstandingApiRpcs() { return cloneMaxOutstandingApiRpcs; @@ -460,26 +244,10 @@ int getMaxLogFlushSeconds() { return maxLogFlushSeconds; } - boolean getRuntimeHttpCompression() { - return runtimeHttpCompression; - } - long getMaxRuntimeLogPerRequest() { return maxRuntimeLogPerRequest; } - boolean getEnableGaeCloudSqlJdbcConnectivity() { - return enableGaeCloudSqlJdbcConnectivity; - } - - boolean getDefaultUseGoogleConnectorj() { - return defaultUseGoogleConnectorj; - } - - boolean getInterruptThreadsFirstOnSoftDeadline() { - return interruptThreadsFirstOnSoftDeadline; - } - boolean getEnableHotspotPerformanceMetrics() { return enableHotspotPerformanceMetrics; } @@ -488,61 +256,21 @@ boolean getUrlfetchDeriveResponseMessage() { return urlfetchDeriveResponseMessage; } - boolean getMailFilenamePreventsInlining() { - return mailFilenamePreventsInlining; - } - - boolean getMailSupportExtendedAttachmentEncodings() { - return mailSupportExtendedAttachmentEncodings; - } - boolean getForceReadaheadOnCloudsqlSocket() { - return forceReadaheadOnCloudsqlSocket; - } - - long getCyclesPerSecond() { - return cyclesPerSecond; - } - - boolean getWaitForDaemonRequestThreads() { - return waitForDaemonRequestThreads; - } boolean getPollForNetwork() { return pollForNetwork; } - boolean getDefaultToNativeUrlStreamHandler() { - return defaultToNativeUrlStreamHandler; - } - boolean getForceUrlfetchUrlStreamHandler() { return forceUrlfetchUrlStreamHandler; } - boolean getEnableSynchronizedAppLogsWriter() { - return enableSynchronizedAppLogsWriter; - } - - boolean getUseEnvVarsFromAppInfo() { - return useEnvVarsFromAppInfo; - } - - boolean getUseJettyHttpProxy() { - return useJettyHttpProxy; - } - int getJettyHttpPort() { return jettyHttpPort; } - int getJettyRequestHeaderSize() { - return jettyRequestHeaderSize; - } - int getJettyResponseHeaderSize() { - return jettyResponseHeaderSize; - } String getFixedApplicationPath() { return fixedApplicationPath; @@ -552,10 +280,6 @@ boolean getDisableApiCallLogging() { return Boolean.getBoolean("disable_api_call_logging_in_apiproxy") || disableApiCallLogging; } - boolean getLogJsonToVarLog() { - return logJsonToVarLog; - } - List getUnknownParams() { return unknownParams; } diff --git a/runtime/impl/src/main/java/com/google/apphosting/runtime/JsonLogHandler.java b/runtime/impl/src/main/java/com/google/apphosting/runtime/JsonLogHandler.java index 98605a99d..322244584 100644 --- a/runtime/impl/src/main/java/com/google/apphosting/runtime/JsonLogHandler.java +++ b/runtime/impl/src/main/java/com/google/apphosting/runtime/JsonLogHandler.java @@ -82,9 +82,7 @@ public void publish(LogRecord record) { private static void appendSpanId(StringBuilder json) { Environment environment = ApiProxy.getCurrentEnvironment(); - if (environment instanceof ApiProxy.EnvironmentWithTrace) { - ApiProxy.EnvironmentWithTrace environmentWithTrace = - (ApiProxy.EnvironmentWithTrace) environment; + if (environment instanceof ApiProxy.EnvironmentWithTrace environmentWithTrace) { environmentWithTrace .getSpanId() .ifPresent(id -> json.append(SPAN_KEY).append("\"").append(id).append("\", ")); @@ -113,9 +111,7 @@ private void appendTraceId(StringBuilder json) { } Environment environment = ApiProxy.getCurrentEnvironment(); - if (environment instanceof ApiProxy.EnvironmentWithTrace) { - ApiProxy.EnvironmentWithTrace environmentWithTrace = - (ApiProxy.EnvironmentWithTrace) environment; + if (environment instanceof ApiProxy.EnvironmentWithTrace environmentWithTrace) { environmentWithTrace .getTraceId() .ifPresent( @@ -135,23 +131,16 @@ private static void appendSeverity(StringBuilder json, LogRecord record) { private static String levelToSeverity(Level level) { int intLevel = (level == null) ? 0 : level.intValue(); - switch (intLevel) { - case 300: // FINEST - case 400: // FINER - case 500: // FINE - return DEBUG; - case 700: // CONFIG - case 800: // INFO - // Java's CONFIG is lower than its INFO, while Stackdriver's NOTICE is greater than its - // INFO. So despite the similarity, we don't try to use NOTICE for CONFIG. - return INFO; - case 900: // WARNING - return WARNING; - case 1000: // SEVERE - return ERROR; - default: - return DEFAULT; - } + return switch (intLevel) { + case 300, 400, 500 -> DEBUG; // FINEST, FINER, FINE + case 700, 800 -> + // Java's CONFIG is lower than its INFO, while Stackdriver's NOTICE is greater than its + // INFO. So despite the similarity, we don't try to use NOTICE for CONFIG. + INFO; // CONFIG, INFO + case 900 -> WARNING; // WARNING + case 1000 -> ERROR; // SEVERE + default -> DEFAULT; + }; } @Override diff --git a/runtime/impl/src/main/java/com/google/apphosting/runtime/NullRpcPlugin.java b/runtime/impl/src/main/java/com/google/apphosting/runtime/NullRpcPlugin.java deleted file mode 100644 index 4c9936987..000000000 --- a/runtime/impl/src/main/java/com/google/apphosting/runtime/NullRpcPlugin.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.apphosting.runtime; - -import com.google.apphosting.runtime.anyrpc.AnyRpcPlugin; -import com.google.apphosting.runtime.anyrpc.CloneControllerServerInterface; -import com.google.apphosting.runtime.anyrpc.EvaluationRuntimeServerInterface; - -/** - * An RPC plugin that does nothing. This is used for the case where the client for - * {@link EvaluationRuntimeServerInterface} and {@link CloneControllerServerInterface} is actually - * in the same process, so there is no RPC. - */ -public class NullRpcPlugin extends AnyRpcPlugin { - public NullRpcPlugin() { - } - - @Override - public void initialize(int serverPort) { - } - - @Override - public void startServer( - EvaluationRuntimeServerInterface evaluationRuntime, - CloneControllerServerInterface cloneController) { - } - - @Override - public boolean serverStarted() { - return true; - } - - @Override - public void blockUntilShutdown() { - } - - @Override - public void stopServer() { - } - - @Override - public void shutdown() { - } - - @Override - public Runnable traceContextPropagating(Runnable runnable) { - // TODO: do we need to do something special? - return runnable; - } -} diff --git a/runtime/impl/src/main/java/com/google/apphosting/runtime/RequestManager.java b/runtime/impl/src/main/java/com/google/apphosting/runtime/RequestManager.java index 6385330e5..26f04e1a5 100644 --- a/runtime/impl/src/main/java/com/google/apphosting/runtime/RequestManager.java +++ b/runtime/impl/src/main/java/com/google/apphosting/runtime/RequestManager.java @@ -48,12 +48,14 @@ import java.text.DateFormat; import java.text.SimpleDateFormat; import java.time.Duration; +import java.util.Locale; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -122,9 +124,7 @@ public class RequestManager implements RequestThreadManager { private final ApiProxyImpl apiProxyImpl; private final boolean threadStopTerminatesClone; private final Map requests; - private final boolean interruptFirstOnSoftDeadline; private int maxOutstandingApiRpcs; - private final boolean waitForDaemonRequestThreads; private final Map environmentVariables; /** Make a partly-initialized builder for a RequestManager. */ @@ -155,18 +155,10 @@ public abstract static class Builder { public abstract boolean threadStopTerminatesClone(); - public abstract Builder setInterruptFirstOnSoftDeadline(boolean x); - - public abstract boolean interruptFirstOnSoftDeadline(); - public abstract Builder setCyclesPerSecond(long x); public abstract long cyclesPerSecond(); - public abstract Builder setWaitForDaemonRequestThreads(boolean x); - - public abstract boolean waitForDaemonRequestThreads(); - public abstract Builder setEnvironment(Map x); public abstract RequestManager build(); @@ -178,9 +170,7 @@ public abstract static class Builder { ApiProxyImpl apiProxyImpl, int maxOutstandingApiRpcs, boolean threadStopTerminatesClone, - boolean interruptFirstOnSoftDeadline, long cyclesPerSecond, - boolean waitForDaemonRequestThreads, ImmutableMap environment) { this.softDeadlineDelay = softDeadlineDelay; this.timerFactory = @@ -189,8 +179,6 @@ public abstract static class Builder { this.apiProxyImpl = apiProxyImpl; this.maxOutstandingApiRpcs = maxOutstandingApiRpcs; this.threadStopTerminatesClone = threadStopTerminatesClone; - this.interruptFirstOnSoftDeadline = interruptFirstOnSoftDeadline; - this.waitForDaemonRequestThreads = waitForDaemonRequestThreads; this.requests = Collections.synchronizedMap(new HashMap()); this.environmentVariables = environment; } @@ -450,7 +438,7 @@ public void sendDeadline(RequestToken token, boolean isUncatchable) { logger.atInfo().log( "Sending deadline: %s, %s, %b", targetThread, token.getRequestId(), isUncatchable); - if (interruptFirstOnSoftDeadline && !isUncatchable) { + if (!isUncatchable) { // Disable thread creation and cancel all pending futures, then interrupt all threads, // all while giving the application some time to return a response after each step. token.getState().setAllowNewRequestThreadCreation(false); @@ -478,7 +466,7 @@ public void sendDeadline(RequestToken token, boolean isUncatchable) { // failed to elicit a response. On hard deadlines, there is no nudging. if (!token.isFinished()) { // SimpleDateFormat isn't threadsafe so just instantiate as-needed - final DateFormat dateFormat = new SimpleDateFormat(SIMPLE_DATE_FORMAT_STRING); + final DateFormat dateFormat = new SimpleDateFormat(SIMPLE_DATE_FORMAT_STRING, Locale.US); // Give the user as much information as we can. final Throwable throwable = createDeadlineThrowable( @@ -629,10 +617,10 @@ private void waitForUserCodeToComplete(RequestToken requestToken) { */ private void attemptThreadPoolShutdown(Collection threads) { for (Thread t : threads) { - if (t instanceof ApiProxyImpl.CurrentRequestThread) { + if (t instanceof ApiProxyImpl.CurrentRequestThread currentRequestThread) { // This thread was made by ThreadManager.currentRequestThreadFactory. Check what Runnable // it was given. - Runnable runnable = ((ApiProxyImpl.CurrentRequestThread) t).userRunnable(); + Runnable runnable = currentRequestThread.userRunnable(); if (runnable.getClass().getName() .equals("java.util.concurrent.ThreadPoolExecutor$Worker")) { // This is the class that ThreadPoolExecutor threads use as their Runnable. @@ -646,8 +634,7 @@ private void attemptThreadPoolShutdown(Collection threads) { Field outerField = runnable.getClass().getDeclaredField("this$0"); outerField.setAccessible(true); Object outer = outerField.get(runnable); - if (outer instanceof ThreadPoolExecutor) { - ThreadPoolExecutor executor = (ThreadPoolExecutor) outer; + if (outer instanceof ThreadPoolExecutor executor) { executor.shutdown(); // We might already have seen this executor via another thread in the loop, but // there's no harm in telling it more than once to shut down. @@ -726,24 +713,18 @@ private void waitForResponseDuringSoftDeadline(Duration responseWaitTimeMs) { */ private Set getActiveThreads(RequestToken token) { Collection threads; - if (waitForDaemonRequestThreads) { - // Join all request threads created using the current request ThreadFactory, including - // daemon ones. - threads = token.getState().requestThreads(); - } else { - // Join all live non-daemon request threads created using the current request ThreadFactory. - Set nonDaemonThreads = new LinkedHashSet<>(); - for (Thread thread : token.getState().requestThreads()) { - if (thread.isDaemon()) { - logger.atInfo().log("Ignoring daemon thread: %s", thread); - } else if (!thread.isAlive()) { - logger.atInfo().log("Ignoring dead thread: %s", thread); - } else { - nonDaemonThreads.add(thread); - } + // Join all live non-daemon request threads created using the current request ThreadFactory. + Set nonDaemonThreads = new LinkedHashSet<>(); + for (Thread thread : token.getState().requestThreads()) { + if (thread.isDaemon()) { + logger.atInfo().log("Ignoring daemon thread: %s", thread); + } else if (!thread.isAlive()) { + logger.atInfo().log("Ignoring dead thread: %s", thread); + } else { + nonDaemonThreads.add(thread); } - threads = nonDaemonThreads; } + threads = nonDaemonThreads; Set activeThreads = new LinkedHashSet<>(threads); activeThreads.remove(Thread.currentThread()); return activeThreads; @@ -836,7 +817,8 @@ private void checkForDeadlocks(final RequestToken token) { long[] deadlockedThreadsIds = THREAD_MX.findDeadlockedThreads(); if (deadlockedThreadsIds != null) { StringBuilder builder = new StringBuilder(); - builder.append("Detected a deadlock across " + deadlockedThreadsIds.length + " threads:"); + builder.append("Detected a deadlock across ").append(deadlockedThreadsIds.length) + .append(" threads:"); for (ThreadInfo info : THREAD_MX.getThreadInfo(deadlockedThreadsIds, MAXIMUM_DEADLOCK_STACK_LENGTH)) { builder.append(info); @@ -858,7 +840,8 @@ private void logMemoryStats() { private void logAllStackTraces() { long[] allthreadIds = THREAD_MX.getAllThreadIds(); StringBuilder builder = new StringBuilder(); - builder.append("Dumping thread info for all " + allthreadIds.length + " runtime threads:"); + builder.append("Dumping thread info for all ").append(allthreadIds.length) + .append(" runtime threads:"); for (ThreadInfo info : THREAD_MX.getThreadInfo(allthreadIds, MAXIMUM_DEADLOCK_STACK_LENGTH)) { builder.append(info); builder.append("\n"); diff --git a/runtime/impl/src/main/java/com/google/apphosting/runtime/RequestRunner.java b/runtime/impl/src/main/java/com/google/apphosting/runtime/RequestRunner.java index 2da9d7284..e2bdc7706 100644 --- a/runtime/impl/src/main/java/com/google/apphosting/runtime/RequestRunner.java +++ b/runtime/impl/src/main/java/com/google/apphosting/runtime/RequestRunner.java @@ -55,7 +55,6 @@ public class RequestRunner implements Runnable { private final UPRequestHandler upRequestHandler; private final RequestManager requestManager; private final BackgroundRequestCoordinator coordinator; - private final boolean compressResponse; private final AppVersion appVersion; private final AnyRpcServerContext rpc; private final UPRequest upRequest; @@ -77,8 +76,6 @@ public abstract static class Builder { public abstract Builder setCoordinator(BackgroundRequestCoordinator coordinator); - public abstract Builder setCompressResponse(boolean compressResponse); - public abstract Builder setAppVersion(AppVersion appVersion); public abstract Builder setRpc(AnyRpcServerContext rpc); @@ -94,7 +91,6 @@ public RequestRunner( UPRequestHandler upRequestHandler, RequestManager requestManager, BackgroundRequestCoordinator coordinator, - boolean compressResponse, AppVersion appVersion, AnyRpcServerContext rpc, UPRequest upRequest, @@ -102,7 +98,6 @@ public RequestRunner( this.upRequestHandler = upRequestHandler; this.requestManager = requestManager; this.coordinator = coordinator; - this.compressResponse = compressResponse; this.appVersion = appVersion; this.rpc = rpc; this.upRequest = upRequest; @@ -348,18 +343,16 @@ public void run() { private void dispatchServletRequest() throws Exception { upRequestHandler.serviceRequest(upRequest, upResponse); - if (compressResponse) { - // try to compress if necessary (http://b/issue?id=3368468) - try { - HttpCompression compression = new HttpCompression(); - compression.attemptCompression(upRequest, upResponse); - } catch (IOException ex) { - // Zip compression did not work... Response is not compressed. - logger.atWarning().withCause(ex).log("Error attempting the compression of the response."); - } catch (RuntimeException ex) { - // To be on the safe side and keep the request ok - logger.atWarning().withCause(ex).log("Error attempting the compression of the response."); - } + // try to compress if necessary (http://b/issue?id=3368468) + try { + HttpCompression compression = new HttpCompression(); + compression.attemptCompression(upRequest, upResponse); + } catch (IOException ex) { + // Zip compression did not work... Response is not compressed. + logger.atWarning().withCause(ex).log("Error attempting the compression of the response."); + } catch (RuntimeException ex) { + // To be on the safe side and keep the request ok + logger.atWarning().withCause(ex).log("Error attempting the compression of the response."); } } diff --git a/runtime/impl/src/main/java/com/google/apphosting/runtime/ServletEngineAdapter.java b/runtime/impl/src/main/java/com/google/apphosting/runtime/ServletEngineAdapter.java index 805abb874..9a7b18126 100644 --- a/runtime/impl/src/main/java/com/google/apphosting/runtime/ServletEngineAdapter.java +++ b/runtime/impl/src/main/java/com/google/apphosting/runtime/ServletEngineAdapter.java @@ -55,12 +55,6 @@ public interface ServletEngineAdapter extends UPRequestHandler { */ void addAppVersion(AppVersion appVersion) throws FileNotFoundException; - /** - * Remove the specified application version and free up any - * resources associated with it. - */ - void deleteAppVersion(AppVersion appVersion); - /** * Sets the {@link SessionStoreFactory} that will be used to create the list * of {@link SessionStore}s to which the HTTP Session will be @@ -75,9 +69,6 @@ public interface ServletEngineAdapter extends UPRequestHandler { */ @AutoValue abstract class Config { - /** Boolean to turn on the Jetty HTTP server. False by default. */ - public abstract boolean useJettyHttpProxy(); - /** * Base root area for a given application. The exploded web app can be located under the * appId/appVersion directory, to be fully compatible with GAE, or given as a Java runtime flag. @@ -113,7 +104,6 @@ abstract class Config { /** Returns an {@code Config.Builder}. */ public static Builder builder() { return new AutoValue_ServletEngineAdapter_Config.Builder() - .setUseJettyHttpProxy(false) .setJettyHttpAddress(HostAndPort.fromParts("::", 8080)) .setJettyReusePort(false) .setJettyRequestHeaderSize(16384) @@ -125,8 +115,6 @@ public static Builder builder() { /** Builder for {@code Config} instances. */ @AutoValue.Builder public abstract static class Builder { - public abstract Builder setUseJettyHttpProxy(boolean useJettyHttpProxy); - public abstract Builder setApplicationRoot(String applicationRoot); public abstract Builder setFixedApplicationPath(@Nullable String applicationpath); diff --git a/runtime/impl/src/main/java/com/google/apphosting/runtime/anyrpc/AnyRpcPlugin.java b/runtime/impl/src/main/java/com/google/apphosting/runtime/anyrpc/AnyRpcPlugin.java deleted file mode 100644 index e12d2738d..000000000 --- a/runtime/impl/src/main/java/com/google/apphosting/runtime/anyrpc/AnyRpcPlugin.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.apphosting.runtime.anyrpc; - -/** - * Base class for RPC-specific plugins. - */ -public abstract class AnyRpcPlugin { - protected AnyRpcPlugin() {} - - /** - * Initializes the plugin, possibly creating any sockets/files/channels/connections - * needed. - * - * @param serverPort the port to listen for RPCs on. - */ - public abstract void initialize(int serverPort); - - /** - * Starts the server using the two specified implementations of the - * RPC-agnostic EvaluationRuntime and CloneController interfaces. - * - * @param evaluationRuntime the evaluation runtime service implementation - * @param cloneController the clone controller service implementation - */ - public abstract void startServer( - EvaluationRuntimeServerInterface evaluationRuntime, - CloneControllerServerInterface cloneController); - - /** - * Returns true if the server has been started and has not stopped. - */ - public abstract boolean serverStarted(); - - /** - * Runs the main loop for the server and waits for the server to shutdown. - * - *

This method MUST be called from the same thread that called - * {@link #startServer(EvaluationRuntimeServerInterface, CloneControllerServerInterface)}. - * - * @throws AssertionError if {@code startServer} was not called or didn't complete - * successfully - */ - public abstract void blockUntilShutdown(); - - /** - * Stops the server. - * - * @throws AssertionError if {@code startServer} was not called or didn't complete - * successfully - */ - public abstract void stopServer(); - - // These methods are Used by JavaRuntimeFactory to implement the CloneController.Callback - // interface, which abstracts over some otherwise RPC-specific functionality. - - /** - * Performs any actions that are necessary to shut down any clients or servers started from this - * plugin. - */ - public abstract void shutdown(); - - /** - * Wraps the provided {@code Runnable} so as to assure propagation of the tracing context. - * - * @param runnable the Runnable to wrap - * @return a Runnable that sets up propagation of the tracing context and then invokes - * the original one - */ - public abstract Runnable traceContextPropagating(Runnable runnable); -} diff --git a/runtime/impl/src/main/java/com/google/apphosting/runtime/anyrpc/CloneControllerServerInterface.java b/runtime/impl/src/main/java/com/google/apphosting/runtime/anyrpc/CloneControllerServerInterface.java deleted file mode 100644 index a0477e1ed..000000000 --- a/runtime/impl/src/main/java/com/google/apphosting/runtime/anyrpc/CloneControllerServerInterface.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.apphosting.runtime.anyrpc; - -import com.google.apphosting.base.protos.ClonePb.CloneSettings; -import com.google.apphosting.base.protos.EmptyMessage; -import com.google.apphosting.base.protos.ModelClonePb.DeadlineInfo; -import com.google.apphosting.base.protos.ModelClonePb.PerformanceDataRequest; - -/** - * RPC-agnostic equivalent of CloneController.ServerInterface. - * - *

If you add methods to CloneController in apphosting/sandbox/clone.proto, - * you need to remember to add them here too. - */ -public interface CloneControllerServerInterface { - void waitForSandbox(AnyRpcServerContext ctx, EmptyMessage req); - - void applyCloneSettings(AnyRpcServerContext ctx, CloneSettings req); - - void sendDeadline(AnyRpcServerContext ctx, DeadlineInfo req); - - void getPerformanceData(AnyRpcServerContext ctx, PerformanceDataRequest req); -} diff --git a/runtime/impl/src/main/java/com/google/apphosting/runtime/anyrpc/EvaluationRuntimeServerInterface.java b/runtime/impl/src/main/java/com/google/apphosting/runtime/anyrpc/EvaluationRuntimeServerInterface.java index 3e178ba25..d1344c773 100644 --- a/runtime/impl/src/main/java/com/google/apphosting/runtime/anyrpc/EvaluationRuntimeServerInterface.java +++ b/runtime/impl/src/main/java/com/google/apphosting/runtime/anyrpc/EvaluationRuntimeServerInterface.java @@ -29,6 +29,4 @@ public interface EvaluationRuntimeServerInterface { void handleRequest(AnyRpcServerContext ctx, UPRequest req); void addAppVersion(AnyRpcServerContext ctx, AppInfo req); - - void deleteAppVersion(AnyRpcServerContext ctx, AppInfo req); } diff --git a/runtime/impl/src/test/java/com/google/apphosting/runtime/ApiDeadlineOracleTest.java b/runtime/impl/src/test/java/com/google/apphosting/runtime/ApiDeadlineOracleTest.java index 45ea4f1d6..9ec859848 100644 --- a/runtime/impl/src/test/java/com/google/apphosting/runtime/ApiDeadlineOracleTest.java +++ b/runtime/impl/src/test/java/com/google/apphosting/runtime/ApiDeadlineOracleTest.java @@ -33,40 +33,32 @@ public class ApiDeadlineOracleTest { @Before public void setUp() throws Exception { - oracle = - new ApiDeadlineOracle.Builder() - .initDeadlineMap(5, "high:60", 10, "high:120") - .initOfflineDeadlineMap(15, "high:600", 20, "high:1200") - .build(); + oracle = new ApiDeadlineOracle.Builder().initDeadlineMap().build(); } @Test public void testDeadline_Default() { - assertThat(oracle.getDeadline("foo", false, null)).isEqualTo(5.0); - assertThat(oracle.getDeadline("foo", true, null)).isEqualTo(15.0); + assertThat(oracle.getDeadline("foo", false, null)).isEqualTo(10.0); + assertThat(oracle.getDeadline("foo", true, null)).isEqualTo(5.0); assertThat(oracle.getDeadline("foo", false, 50)).isEqualTo(10.0); - assertThat(oracle.getDeadline("foo", true, 50)).isEqualTo(20.0); + assertThat(oracle.getDeadline("foo", true, 50)).isEqualTo(10.0); assertThat(oracle.getDeadline("foo", false, 1)).isEqualTo(1.0); assertThat(oracle.getDeadline("foo", true, 1)).isEqualTo(1.0); } @Test public void testDeadline_InitialPackageOverride() { - assertThat(oracle.getDeadline("high", false, null)).isEqualTo(60.0); - assertThat(oracle.getDeadline("high", true, null)).isEqualTo(600.0); - assertThat(oracle.getDeadline("high", false, 3600)).isEqualTo(120.0); - assertThat(oracle.getDeadline("high", true, 3600)).isEqualTo(1200.0); + // "blobstore" is one of the packages with specific deadlines in the hardcoded map. + assertThat(oracle.getDeadline("blobstore", false, null)).isEqualTo(15.0); + assertThat(oracle.getDeadline("blobstore", true, null)).isEqualTo(15.0); + assertThat(oracle.getDeadline("blobstore", false, 3600)).isEqualTo(30.0); + assertThat(oracle.getDeadline("blobstore", true, 3600)).isEqualTo(30.0); } @Test public void testDeadline_LaterPackageOverride() { oracle.addPackageDefaultDeadline("foo", 30); oracle.addPackageMaxDeadline("foo", 300); - oracle.addOfflinePackageDefaultDeadline("foo", 60); - oracle.addOfflinePackageMaxDeadline("foo", 600); assertThat(oracle.getDeadline("foo", false, null)).isEqualTo(30.0); - assertThat(oracle.getDeadline("foo", true, null)).isEqualTo(60.0); - assertThat(oracle.getDeadline("foo", false, 3600)).isEqualTo(300.0); - assertThat(oracle.getDeadline("foo", true, 3600)).isEqualTo(600.0); } } diff --git a/runtime/impl/src/test/java/com/google/apphosting/runtime/ApiProxyImplTest.java b/runtime/impl/src/test/java/com/google/apphosting/runtime/ApiProxyImplTest.java index 1509f11b8..19530a272 100644 --- a/runtime/impl/src/test/java/com/google/apphosting/runtime/ApiProxyImplTest.java +++ b/runtime/impl/src/test/java/com/google/apphosting/runtime/ApiProxyImplTest.java @@ -97,8 +97,8 @@ @RunWith(JUnit4.class) public class ApiProxyImplTest { - private static final double DEFAULT_API_DEADLINE = 5.0; - private static final double DEFAULT_OFFLINE_API_DEADLINE = 7.0; + private static final double DEFAULT_API_DEADLINE = 10.0; + private static final double DEFAULT_OFFLINE_API_DEADLINE = 5.0; private static final double MAX_API_DEADLINE = 10.0; private static final String APP_ID = "app123"; private static final String ENGINE_ID = "non-default"; @@ -138,14 +138,7 @@ public void setUp() throws IOException { rootDirectory = temporaryFolder.newFolder("appengine" + System.nanoTime()); maxConcurrentApiCalls = 10; oracle = - new ApiDeadlineOracle.Builder() - .initDeadlineMap( - DEFAULT_API_DEADLINE, "", - MAX_API_DEADLINE, "") - .initOfflineDeadlineMap( - DEFAULT_OFFLINE_API_DEADLINE, "", - MAX_API_DEADLINE, "") - .build(); + new ApiDeadlineOracle.Builder().initDeadlineMap().build(); sleepSemaphore = new Semaphore(0); APIHostClientInterface apiHost = createAPIHost(); delegate = @@ -363,39 +356,8 @@ public void testCurrentEnvironment() { assertThat(environment.getAttributes().get(ApiProxyImpl.APPSERVER_TASK_BNS)).isEqualTo(null); assertThat( environment.getAttributes().get(ApiProxyImpl.CLOUD_SQL_JDBC_CONNECTIVITY_ENABLED_KEY)) - .isEqualTo(false); - assertThat(environment.getTraceId()).isEmpty(); - } - - @Test - public void testCloudSqlJdbcConnectivityEnabled() { - APIHostClientInterface apiHost = createAPIHost(); - delegate = - ApiProxyImpl.builder() - .setApiHost(apiHost) - .setDeadlineOracle(oracle) - .setByteCountBeforeFlushing(BYTE_COUNT_BEFORE_FLUSHING) - .setMaxLogLineSize(MAX_LOG_LINE_SIZE) - .setCloudSqlJdbcConnectivityEnabled(true) - .build(); - assertThat( - createEnvironment() - .getAttributes() - .get(ApiProxyImpl.CLOUD_SQL_JDBC_CONNECTIVITY_ENABLED_KEY)) .isEqualTo(true); - - delegate = - ApiProxyImpl.builder() - .setApiHost(apiHost) - .setDeadlineOracle(oracle) - .setByteCountBeforeFlushing(BYTE_COUNT_BEFORE_FLUSHING) - .setMaxLogLineSize(MAX_LOG_LINE_SIZE) - .build(); - assertThat( - createEnvironment() - .getAttributes() - .get(ApiProxyImpl.CLOUD_SQL_JDBC_CONNECTIVITY_ENABLED_KEY)) - .isEqualTo(false); + assertThat(environment.getTraceId()).isEmpty(); } @Test @@ -1348,7 +1310,7 @@ public void testAsync_deadlineExceededWhileWaitingForApiSlot() throws Exception // seconds while the first API call was using that slot. So if elapsed time is < 2 seconds then // that didn't happen. Instant asyncCallEnd = Instant.now(); - assertThat(Duration.between(asyncCallStart, asyncCallEnd).getSeconds()).isLessThan(2); + assertThat(Duration.between(asyncCallStart, asyncCallEnd).toSeconds()).isLessThan(2); // Give it a couple of seconds to time out. If we don't respect the timeout while waiting for // the API slot, then we won't succeed in timing out. diff --git a/runtime/runtime_impl_jetty9/src/test/java/com/google/apphosting/runtime/jetty9/AppInfoFactoryTest.java b/runtime/impl/src/test/java/com/google/apphosting/runtime/AppInfoFactoryTest.java similarity index 87% rename from runtime/runtime_impl_jetty9/src/test/java/com/google/apphosting/runtime/jetty9/AppInfoFactoryTest.java rename to runtime/impl/src/test/java/com/google/apphosting/runtime/AppInfoFactoryTest.java index 06f1b4436..e575cb94c 100644 --- a/runtime/runtime_impl_jetty9/src/test/java/com/google/apphosting/runtime/jetty9/AppInfoFactoryTest.java +++ b/runtime/impl/src/test/java/com/google/apphosting/runtime/AppInfoFactoryTest.java @@ -14,24 +14,18 @@ * limitations under the License. */ -package com.google.apphosting.runtime.jetty9; +package com.google.apphosting.runtime; import static com.google.common.base.StandardSystemProperty.USER_DIR; import static com.google.common.truth.Truth.assertThat; -import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertThrows; import com.google.appengine.tools.development.resource.ResourceExtractor; import com.google.apphosting.base.protos.AppinfoPb; -import com.google.apphosting.utils.config.AppYaml; import com.google.common.collect.ImmutableMap; -import java.io.File; -import java.io.FileInputStream; import java.io.IOException; -import java.io.InputStreamReader; import java.nio.file.NoSuchFileException; import java.nio.file.Path; -import java.nio.file.Paths; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -53,16 +47,15 @@ public final class AppInfoFactoryTest { @Before public void setUp() throws IOException { - Path projPath = Paths.get(temporaryFolder.newFolder(PROJECT_RESOURCE_NAME).getPath()); + Path projPath = Path.of(temporaryFolder.newFolder(PROJECT_RESOURCE_NAME).getPath()); appRoot = projPath.getParent().toString(); - fixedAppDir = Paths.get(projPath.toString(), "100.mydeployment").toString(); + fixedAppDir = Path.of(projPath.toString(), "100.mydeployment").toString(); ResourceExtractor.toFile(PROJECT_RESOURCE_NAME, projPath.toString()); } @Test public void getGaeService_nonDefault() throws Exception { - AppInfoFactory factory = - new AppInfoFactory(ImmutableMap.of("GAE_SERVICE", "mytestservice")); + AppInfoFactory factory = new AppInfoFactory(ImmutableMap.of("GAE_SERVICE", "mytestservice")); assertThat(factory.getGaeService()).isEqualTo("mytestservice"); } @@ -140,7 +133,6 @@ public void getAppInfo_fixedApplicationPath() throws Exception { assertThat(appInfo.getAppId()).isEqualTo("s~myapp"); assertThat(appInfo.getVersionId()).isEqualTo("mytestservice:100.mydeployment"); - assertThat(appInfo.getRuntimeId()).isEqualTo("java8"); } @Test @@ -157,7 +149,6 @@ public void getAppInfo_appRoot() throws Exception { assertThat(appInfo.getAppId()).isEqualTo("s~myapp"); assertThat(appInfo.getVersionId()).isEqualTo("mytestservice:100.mydeployment"); - assertThat(appInfo.getRuntimeId()).isEqualTo("java8"); } @Test @@ -179,7 +170,7 @@ public void getAppInfo_noAppYaml() throws Exception { assertThat(appInfo.getAppId()).isEqualTo("s~myapp"); assertThat(appInfo.getVersionId()).isEqualTo("mytestservice:100.mydeployment"); - assertThat(appInfo.getRuntimeId()).isEqualTo("java8"); + assertThat(appInfo.getApiVersion()).isEmpty(); } @Test @@ -208,14 +199,10 @@ public void getAppInfo_givenAppYaml() throws Exception { "GAE_APPLICATION", "s~myapp", "GOOGLE_CLOUD_PROJECT", "mytestproject")); - File appYamlFile = new File(fixedAppDir + "/WEB-INF/appengine-generated/app.yaml"); - AppYaml appYaml = AppYaml.parse(new InputStreamReader(new FileInputStream(appYamlFile), UTF_8)); - - AppinfoPb.AppInfo appInfo = factory.getAppInfoFromAppYaml(appYaml); + AppinfoPb.AppInfo appInfo = factory.getAppInfo(); assertThat(appInfo.getAppId()).isEqualTo("s~myapp"); assertThat(appInfo.getVersionId()).isEqualTo("mytestservice:100.mydeployment"); - assertThat(appInfo.getRuntimeId()).isEqualTo("java8"); } @Test @@ -233,6 +220,5 @@ public void getAppInfo_givenVersion() throws Exception { assertThat(appInfo.getAppId()).isEqualTo("s~myapp"); assertThat(appInfo.getVersionId()).isEqualTo("mytestservice:100.mydeployment"); - assertThat(appInfo.getRuntimeId()).isEqualTo("java8"); } } diff --git a/runtime/impl/src/test/java/com/google/apphosting/runtime/ApplicationEnvironmentTest.java b/runtime/impl/src/test/java/com/google/apphosting/runtime/ApplicationEnvironmentTest.java index 4a4dc0d00..bd32a3b97 100644 --- a/runtime/impl/src/test/java/com/google/apphosting/runtime/ApplicationEnvironmentTest.java +++ b/runtime/impl/src/test/java/com/google/apphosting/runtime/ApplicationEnvironmentTest.java @@ -27,7 +27,13 @@ /** Unit tests for the {@link ApplicationEnvironment} class. */ @RunWith(JUnit4.class) public class ApplicationEnvironmentTest { - private void getUseGoogleConnectorJDoesNotRequireSetterFuzz(boolean val) throws Exception { + + /** + * Verifies the getUseGoogleConnectorJ method does not require a call to the corresponding setter + * before it honors the constructor defaults. + */ + @Test + public void getUseGoogleConnectorJ_doesNotRequireSetter() throws Exception { ApplicationEnvironment ae = new ApplicationEnvironment( "an_appId", @@ -35,28 +41,14 @@ private void getUseGoogleConnectorJDoesNotRequireSetterFuzz(boolean val) throws ImmutableMap.of("a_sysprop_name", "a_sysprop_value"), ImmutableMap.of("an_env_name", "an_env_value"), new File("/foo/bar"), - ApplicationEnvironment.RuntimeConfiguration.builder() - .setCloudSqlJdbcConnectivityEnabled(true) - .setUseGoogleConnectorJ(val) - .build()); + ApplicationEnvironment.RuntimeConfiguration.builder().build()); - assertThat(ae.getUseGoogleConnectorJ()).isEqualTo(val); + assertThat(ae.getUseGoogleConnectorJ()).isTrue(); - ae.setUseGoogleConnectorJ(Boolean.valueOf(!val)); - assertThat(ae.getUseGoogleConnectorJ()).isEqualTo(!val); + ae.setUseGoogleConnectorJ(Boolean.valueOf(false)); + assertThat(ae.getUseGoogleConnectorJ()).isFalse(); ae.setUseGoogleConnectorJ(null); // go back to the ctor default - assertThat(ae.getUseGoogleConnectorJ()).isEqualTo(val); - } - - /** - * Verifies the getUseGoogleConnectorJ method does not require a call to the corresponding setter - * before it honors the constructor defaults. - */ - @Test - public void getUseGoogleConnectorJ_doesNotRequireSetter() throws Exception { - // Try true and false just for good measure: - getUseGoogleConnectorJDoesNotRequireSetterFuzz(true); - getUseGoogleConnectorJDoesNotRequireSetterFuzz(false); + assertThat(ae.getUseGoogleConnectorJ()).isTrue(); } } diff --git a/runtime/impl/src/test/java/com/google/apphosting/runtime/NullSandboxPluginTest.java b/runtime/impl/src/test/java/com/google/apphosting/runtime/NullSandboxPluginTest.java index 7011f1269..8e3322073 100644 --- a/runtime/impl/src/test/java/com/google/apphosting/runtime/NullSandboxPluginTest.java +++ b/runtime/impl/src/test/java/com/google/apphosting/runtime/NullSandboxPluginTest.java @@ -67,6 +67,7 @@ private ClassPathUtils mockClassPathUtils() { when(classPathUtils.getRuntimeSharedUrls()).thenReturn(new URL[0]); when(classPathUtils.getRuntimeImplUrls()).thenReturn(new URL[0]); when(classPathUtils.getPrebundledUrls()).thenReturn(new URL[0]); + when(classPathUtils.getConnectorJUrls()).thenReturn(new URL[0]); return classPathUtils; } diff --git a/runtime/impl/src/test/resources/com/google/apphosting/runtime/jetty9/mytestproject/100.mydeployment/WEB-INF/appengine-generated/app.yaml b/runtime/impl/src/test/resources/com/google/apphosting/runtime/mytestproject/100.mydeployment/WEB-INF/appengine-generated/app.yaml similarity index 100% rename from runtime/impl/src/test/resources/com/google/apphosting/runtime/jetty9/mytestproject/100.mydeployment/WEB-INF/appengine-generated/app.yaml rename to runtime/impl/src/test/resources/com/google/apphosting/runtime/mytestproject/100.mydeployment/WEB-INF/appengine-generated/app.yaml diff --git a/runtime/lite/pom.xml b/runtime/lite/pom.xml index f1a5bfe8b..ab2ad1662 100644 --- a/runtime/lite/pom.xml +++ b/runtime/lite/pom.xml @@ -61,7 +61,7 @@ com.google.testparameterinjector test-parameter-injector - 1.20 + 1.21 test diff --git a/runtime/lite/src/main/java/com/google/appengine/runtime/lite/AppEngineRuntime.java b/runtime/lite/src/main/java/com/google/appengine/runtime/lite/AppEngineRuntime.java index 7a00e198e..a50953e63 100644 --- a/runtime/lite/src/main/java/com/google/appengine/runtime/lite/AppEngineRuntime.java +++ b/runtime/lite/src/main/java/com/google/appengine/runtime/lite/AppEngineRuntime.java @@ -23,13 +23,13 @@ import com.google.apphosting.base.protos.AppinfoPb; import com.google.apphosting.runtime.ApiDeadlineOracle; import com.google.apphosting.runtime.ApiProxyImpl; +import com.google.apphosting.runtime.AppInfoFactory; import com.google.apphosting.runtime.AppVersion; import com.google.apphosting.runtime.ApplicationEnvironment; import com.google.apphosting.runtime.SessionsConfig; import com.google.apphosting.runtime.anyrpc.APIHostClientInterface; import com.google.apphosting.runtime.http.HttpApiHostClientFactory; import com.google.apphosting.runtime.jetty9.AppEngineWebAppContext; -import com.google.apphosting.runtime.jetty9.AppInfoFactory; import com.google.apphosting.runtime.jetty9.AppVersionHandlerFactory; import com.google.apphosting.runtime.jetty9.JettyServerConnectorWithReusePort; import com.google.apphosting.runtime.jetty9.WebAppContextFactory; @@ -334,7 +334,6 @@ static ApiProxyImpl.Builder makeApiProxyImplBuilder( .setByteCountBeforeFlushing(DEFAULT_FLUSH_APP_LOGS_EVERY_BYTE_COUNT) .setMaxLogFlushTime(MAX_LOG_FLUSH_TIME) .setCoordinator(dispatcher) - .setCloudSqlJdbcConnectivityEnabled(true) .setLogToLogservice(false) .setDisableApiCallLogging(true) .setApiHost(apiHost); @@ -361,8 +360,6 @@ static AppVersion createAppVersion( ApplicationEnvironment.RuntimeConfiguration runtimeConfiguration = ApplicationEnvironment.RuntimeConfiguration.builder() - .setCloudSqlJdbcConnectivityEnabled(true) - .setUseGoogleConnectorJ(true) .build(); ApplicationEnvironment environment = diff --git a/runtime/lite/src/main/java/com/google/appengine/runtime/lite/DeadlineOracleFactory.java b/runtime/lite/src/main/java/com/google/appengine/runtime/lite/DeadlineOracleFactory.java index 7897f1761..4b8ec3dab 100644 --- a/runtime/lite/src/main/java/com/google/appengine/runtime/lite/DeadlineOracleFactory.java +++ b/runtime/lite/src/main/java/com/google/appengine/runtime/lite/DeadlineOracleFactory.java @@ -25,37 +25,33 @@ final class DeadlineOracleFactory { private static final double YEAR_SECONDS = 31536000.0; interface DeadlineSetter { - void set(String pkg, double defOnline, double defOffline); + void set(String pkg, double defOnline); } static ApiDeadlineOracle create() { Map defaultOnlineDeadlineMap = new HashMap<>(); Map maxOnlineDeadlineMap = new HashMap<>(); - Map defaultOfflineDeadlineMap = new HashMap<>(); - Map maxOfflineDeadlineMap = new HashMap<>(); DeadlineSetter setter = - (String pkg, double defOnline, double defOffline) -> { + (String pkg, double defOnline) -> { defaultOnlineDeadlineMap.put(pkg, defOnline); maxOnlineDeadlineMap.put(pkg, YEAR_SECONDS); - defaultOfflineDeadlineMap.put(pkg, defOffline); - maxOfflineDeadlineMap.put(pkg, YEAR_SECONDS); }; - setter.set("app_config_service", 60.0, 60.0); - setter.set("blobstore", 15.0, 15.0); - setter.set("datastore_v3", 60.0, 60.0); - setter.set("datastore_v4", 60.0, 60.0); - setter.set("file", 30.0, 30.0); - setter.set("images", 30.0, 30.0); - setter.set("logservice", 60.0, 60.0); - setter.set("modules", 60.0, 60.0); - setter.set("rdbms", 60.0, 60.0); - setter.set("remote_socket", 60.0, 60.0); - setter.set("search", 10.0, 10.0); - setter.set("stubby", 10.0, 10.0); - setter.set("taskqueue", 10.0, 5.0); - setter.set("urlfetch", 10.0, 5.0); + setter.set("app_config_service", 60.0); + setter.set("blobstore", 15.0); + setter.set("datastore_v3", 60.0); + setter.set("datastore_v4", 60.0); + setter.set("file", 30.0); + setter.set("images", 30.0); + setter.set("logservice", 60.0); + setter.set("modules", 60.0); + setter.set("rdbms", 60.0); + setter.set("remote_socket", 60.0); + setter.set("search", 10.0); + setter.set("stubby", 10.0); + setter.set("taskqueue", 10.0); + setter.set("urlfetch", 10.0); return new ApiDeadlineOracle.Builder() .initDeadlineMap( @@ -64,12 +60,6 @@ static ApiDeadlineOracle create() { defaultOnlineDeadlineMap, /*maxDeadline=*/ YEAR_SECONDS, maxOnlineDeadlineMap)) - .initOfflineDeadlineMap( - new ApiDeadlineOracle.DeadlineMap( - /*defaultDeadline=*/ YEAR_SECONDS, - defaultOfflineDeadlineMap, - /*maxDeadline=*/ YEAR_SECONDS, - maxOfflineDeadlineMap)) .build(); } diff --git a/runtime/lite/src/main/java/com/google/appengine/runtime/lite/LogHandler.java b/runtime/lite/src/main/java/com/google/appengine/runtime/lite/LogHandler.java index 184e5f2ba..f0828f66e 100644 --- a/runtime/lite/src/main/java/com/google/appengine/runtime/lite/LogHandler.java +++ b/runtime/lite/src/main/java/com/google/appengine/runtime/lite/LogHandler.java @@ -89,8 +89,7 @@ protected LogEntry.Builder logEntryFor(LogRecord record) { // Get trace context from the special App Engine SDK location: ApiProxy.Environment environment = ApiProxy.getCurrentEnvironment(); - if (environment instanceof ApiProxy.EnvironmentWithTrace) { - ApiProxy.EnvironmentWithTrace environmentImpl = (ApiProxy.EnvironmentWithTrace) environment; + if (environment instanceof ApiProxy.EnvironmentWithTrace environmentImpl) { environmentImpl .getTraceId() .ifPresent(x -> builder.setTrace("projects/" + PROJECT_ID + "/traces/" + x)); diff --git a/runtime/lite/src/main/java/com/google/appengine/runtime/lite/RequestHandler.java b/runtime/lite/src/main/java/com/google/appengine/runtime/lite/RequestHandler.java index 2c1af16d7..ff5b6501a 100644 --- a/runtime/lite/src/main/java/com/google/appengine/runtime/lite/RequestHandler.java +++ b/runtime/lite/src/main/java/com/google/appengine/runtime/lite/RequestHandler.java @@ -17,9 +17,9 @@ package com.google.appengine.runtime.lite; import com.google.apphosting.runtime.AppVersion; +import com.google.apphosting.runtime.AppInfoFactory; import com.google.apphosting.runtime.MutableUpResponse; import com.google.apphosting.runtime.anyrpc.AnyRpcServerContext; -import com.google.apphosting.runtime.jetty9.AppInfoFactory; import com.google.apphosting.runtime.jetty9.AppVersionHandlerFactory; import com.google.apphosting.runtime.jetty9.UPRequestTranslator; import com.google.common.flogger.GoogleLogger; diff --git a/runtime/lite/src/test/java/com/google/appengine/runtime/lite/AppEngineRuntimeTest.java b/runtime/lite/src/test/java/com/google/appengine/runtime/lite/AppEngineRuntimeTest.java index c57169839..e86493c2b 100644 --- a/runtime/lite/src/test/java/com/google/appengine/runtime/lite/AppEngineRuntimeTest.java +++ b/runtime/lite/src/test/java/com/google/appengine/runtime/lite/AppEngineRuntimeTest.java @@ -35,9 +35,9 @@ import com.google.apphosting.runtime.ApiProxyImpl; import com.google.apphosting.runtime.AppVersion; import com.google.apphosting.runtime.ApplicationEnvironment; +import com.google.apphosting.runtime.AppInfoFactory; import com.google.apphosting.runtime.SessionsConfig; import com.google.apphosting.runtime.http.FakeHttpApiHost; -import com.google.apphosting.runtime.jetty9.AppInfoFactory; import com.google.apphosting.testing.PortPicker; import com.google.common.base.Joiner; import com.google.common.collect.ImmutableMap; @@ -857,11 +857,16 @@ public void makeRequestManagerBuilder_validate() { } private static void validateDeadline( - ApiDeadlineOracle oracle, String packageName, double defOnline, double defOffline) { + ApiDeadlineOracle oracle, + String packageName, + double defOnline, + double maxOnline, + double defOffline, + double maxOffline) { assertThat(oracle.getDeadline(packageName, false, null)).isEqualTo(defOnline); - assertThat(oracle.getDeadline(packageName, false, Double.MAX_VALUE)).isEqualTo(YEAR_SECONDS); + assertThat(oracle.getDeadline(packageName, false, Double.MAX_VALUE)).isEqualTo(maxOnline); assertThat(oracle.getDeadline(packageName, true, null)).isEqualTo(defOffline); - assertThat(oracle.getDeadline(packageName, true, Double.MAX_VALUE)).isEqualTo(YEAR_SECONDS); + assertThat(oracle.getDeadline(packageName, true, Double.MAX_VALUE)).isEqualTo(maxOffline); } @Test @@ -873,26 +878,25 @@ public void makeApiProxyImplBuilder_validate() { AppEngineRuntime.makeApiProxyImplBuilder(apiHostAddress, dispatcher); ApiDeadlineOracle oracle = builder.deadlineOracle(); - validateDeadline(oracle, "app_config_service", 60.0, 60.0); - validateDeadline(oracle, "blobstore", 15.0, 15.0); - validateDeadline(oracle, "datastore_v3", 60.0, 60.0); - validateDeadline(oracle, "datastore_v4", 60.0, 60.0); - validateDeadline(oracle, "file", 30.0, 30.0); - validateDeadline(oracle, "images", 30.0, 30.0); - validateDeadline(oracle, "logservice", 60.0, 60.0); - validateDeadline(oracle, "modules", 60.0, 60.0); - validateDeadline(oracle, "rdbms", 60.0, 60.0); - validateDeadline(oracle, "remote_socket", 60.0, 60.0); - validateDeadline(oracle, "search", 10.0, 10.0); - validateDeadline(oracle, "stubby", 10.0, 10.0); - validateDeadline(oracle, "taskqueue", 10.0, 5.0); - validateDeadline(oracle, "urlfetch", 10.0, 5.0); + validateDeadline(oracle, "app_config_service", 60.0, YEAR_SECONDS, 60.0, 60.0); + validateDeadline(oracle, "blobstore", 15.0, YEAR_SECONDS, 15.0, 30.0); + validateDeadline(oracle, "datastore_v3", 60.0, YEAR_SECONDS, 60.0, 270.0); + validateDeadline(oracle, "datastore_v4", 60.0, YEAR_SECONDS, 60.0, 270.0); + validateDeadline(oracle, "file", 30.0, YEAR_SECONDS, 30.0, 60.0); + validateDeadline(oracle, "images", 30.0, YEAR_SECONDS, 30.0, 30.0); + validateDeadline(oracle, "logservice", 60.0, YEAR_SECONDS, 60.0, 60.0); + validateDeadline(oracle, "modules", 60.0, YEAR_SECONDS, 60.0, 60.0); + validateDeadline(oracle, "rdbms", 60.0, YEAR_SECONDS, 60.0, 600.0); + validateDeadline(oracle, "remote_socket", 60.0, YEAR_SECONDS, 60.0, 60.0); + validateDeadline(oracle, "search", 10.0, YEAR_SECONDS, 10.0, 60.0); + validateDeadline(oracle, "stubby", 10.0, YEAR_SECONDS, 10.0, 600.0); + validateDeadline(oracle, "taskqueue", 10.0, YEAR_SECONDS, 5.0, 30.0); + validateDeadline(oracle, "urlfetch", 10.0, YEAR_SECONDS, 5.0, 600.0); assertThat(builder.coordinator()).isSameInstanceAs(dispatcher); assertThat(builder.externalDatacenterName()).isEqualTo("MARS"); assertThat(builder.byteCountBeforeFlushing()).isEqualTo(100 * 1024L); assertThat(builder.maxLogFlushTime()).isEqualTo(Duration.ofMinutes(1)); - assertThat(builder.cloudSqlJdbcConnectivityEnabled()).isTrue(); assertThat(builder.disableApiCallLogging()).isTrue(); } @@ -936,9 +940,6 @@ public void createAppVersion_withLegacy() throws Exception { assertThat(appEnv.getEnvironmentVariables()).isEmpty(); assertThat(appEnv.getUseGoogleConnectorJ()).isTrue(); - ApplicationEnvironment.RuntimeConfiguration runtimeConfig = appEnv.getRuntimeConfiguration(); - assertThat(runtimeConfig.getCloudSqlJdbcConnectivityEnabled()).isTrue(); - assertThat(runtimeConfig.getUseGoogleConnectorJ()).isTrue(); } @Test @@ -976,10 +977,6 @@ public void createAppVersion_noLegacy() throws Exception { assertThat(appEnv.getEnvironmentVariables()).isEmpty(); assertThat(appEnv.getUseGoogleConnectorJ()).isTrue(); - ApplicationEnvironment.RuntimeConfiguration runtimeConfig = appEnv.getRuntimeConfiguration(); - assertThat(runtimeConfig.getCloudSqlJdbcConnectivityEnabled()).isTrue(); - assertThat(runtimeConfig.getUseGoogleConnectorJ()).isTrue(); - SessionsConfig sessionsConfig = appVersion.getSessionsConfig(); assertThat(sessionsConfig.isEnabled()).isFalse(); assertThat(sessionsConfig.isAsyncPersistence()).isFalse(); diff --git a/runtime/local_jetty121/pom.xml b/runtime/local_jetty121/pom.xml index 56ef2385c..2fcceb0fb 100644 --- a/runtime/local_jetty121/pom.xml +++ b/runtime/local_jetty121/pom.xml @@ -127,7 +127,7 @@ org.eclipse.jetty.toolchain jetty-servlet-api - 4.0.6 + 4.0.9 org.eclipse.jetty diff --git a/runtime/local_jetty121/src/main/java/com/google/appengine/tools/development/jetty/JettyContainerService.java b/runtime/local_jetty121/src/main/java/com/google/appengine/tools/development/jetty/JettyContainerService.java index 7d6ffd9b6..086d4c107 100644 --- a/runtime/local_jetty121/src/main/java/com/google/appengine/tools/development/jetty/JettyContainerService.java +++ b/runtime/local_jetty121/src/main/java/com/google/appengine/tools/development/jetty/JettyContainerService.java @@ -444,17 +444,14 @@ protected void startHotDeployScanner() throws Exception { scanner.setScanInterval(SCAN_INTERVAL_SECONDS); scanner.setScanDirs(ImmutableList.of(getScanTarget().toPath())); scanner.setFilenameFilter( - new FilenameFilter() { - @Override - public boolean accept(File dir, String name) { - try { - if (name.equals(getScanTarget().getName())) { - return true; - } - return false; - } catch (Exception e) { - return false; + (dir, name) -> { + try { + if (name.equals(getScanTarget().getName())) { + return true; } + return false; + } catch (Exception e) { + return false; } }); scanner.addListener(new ScannerListener()); diff --git a/runtime/local_jetty121_ee11/pom.xml b/runtime/local_jetty121_ee11/pom.xml index 33cb17b55..f8fb7ca4e 100644 --- a/runtime/local_jetty121_ee11/pom.xml +++ b/runtime/local_jetty121_ee11/pom.xml @@ -127,7 +127,7 @@ org.eclipse.jetty.toolchain jetty-servlet-api - 4.0.6 + 4.0.9 org.eclipse.jetty diff --git a/runtime/local_jetty121_ee11/src/main/java/com/google/appengine/tools/development/jetty/ee11/JettyContainerService.java b/runtime/local_jetty121_ee11/src/main/java/com/google/appengine/tools/development/jetty/ee11/JettyContainerService.java index 179838bc6..5f1b33283 100644 --- a/runtime/local_jetty121_ee11/src/main/java/com/google/appengine/tools/development/jetty/ee11/JettyContainerService.java +++ b/runtime/local_jetty121_ee11/src/main/java/com/google/appengine/tools/development/jetty/ee11/JettyContainerService.java @@ -451,17 +451,11 @@ protected void startHotDeployScanner() throws Exception { scanner.setScanInterval(SCAN_INTERVAL_SECONDS); scanner.setScanDirs(ImmutableList.of(getScanTarget().toPath())); scanner.setFilenameFilter( - new FilenameFilter() { - @Override - public boolean accept(File dir, String name) { - try { - if (name.equals(getScanTarget().getName())) { - return true; - } - return false; - } catch (Exception e) { - return false; - } + (dir, name) -> { + try { + return name.equals(getScanTarget().getName()); + } catch (Exception e) { + return false; } }); scanner.addListener(new ScannerListener()); diff --git a/runtime/main/src/main/java/com/google/apphosting/runtime/JavaRuntimeMainWithDefaults.java b/runtime/main/src/main/java/com/google/apphosting/runtime/JavaRuntimeMainWithDefaults.java index 3cc9b9560..fd692bfe8 100644 --- a/runtime/main/src/main/java/com/google/apphosting/runtime/JavaRuntimeMainWithDefaults.java +++ b/runtime/main/src/main/java/com/google/apphosting/runtime/JavaRuntimeMainWithDefaults.java @@ -86,89 +86,8 @@ private static List getRuntimeFlags() { ? System.getenv().getOrDefault(API_HOSTPORT_ENV_VARIABLE_NAME, "localhost:8089") : "appengine.googleapis.internal:10001"; return Arrays.asList( - "--api_call_deadline=10", - "--api_call_deadline_map=app_config_service:60.0," - + "blobstore:15.0," - + "datastore_v3:60.0," - + "datastore_v4:60.0,file:30.0," - + "images:30.0," - + "logservice:60.0," - + "mail:30.0," - + "modules:60.0," - + "rdbms:60.0," - + "remote_socket:60.0," - + "search:10.0," - + "stubby:10.0", - "--appengine_release_name=mainwithdefaults", - "--application_root=notused", - "--cycles_per_second=1000000000", - "--default_to_builtin_url_stream_handler=true", - "--default_use_google_connectorj=true", - "--enable_gae_cloud_sql_jdbc_connectivity=true", - "--enable_synchronized_app_logs_writer=true", - "--external_datacenter_name=MARS", - // This flag must be given in this Main. - // "--fixed_application_path=" + ..., - "--force_readahead_on_cloudsql_socket=true", - "--interrupt_threads_first_on_soft_deadline=true", - "--java_soft_deadline_ms=10600", "--jetty_http_port=" + jettyPort, - "--jetty_request_header_size=262144", - "--jetty_response_header_size=262144", - "--mail_filename_prevents_inlining=true", - "--mail_support_extended_attachment_encodings=true", - "--max_api_call_deadline_map=app_config_service:60.0," - + "blobstore:30.0," - + "datastore_v3:270.0," - + "datastore_v4:270.0," - + "file:60.0," - + "images:30.0," - + "logservice:60.0," - + "mail:60.0," - + "modules:60.0," - + "rdbms:60.0," - + "remote_socket:60.0," - + "search:60.0," - + "stubby:60.0," - + "taskqueue:30.0," - + "urlfetch:60.0", - "--max_offline_api_call_deadline=10", - "--max_offline_api_call_deadline_map=app_config_service:60.0," - + "blobstore:30.0," - + "datastore_v3:270.0," - + "datastore_v4:270.0," - + "file:60.0," - + "images:30.0," - + "logservice:60.0," - + "mail:60.0," - + "modules:60.0,rdbms:600.0," - + "remote_socket:60.0," - + "search:60.0," - + "stubby:600.0," - + "taskqueue:30.0," - + "urlfetch:600.0", - "--offline_api_call_deadline=5", - "--offline_api_call_deadline_map=app_config_service:60.0," - + "blobstore:15.0," - + "datastore_v3:60.0," - + "datastore_v4:60.0," - + "file:30.0," - + "images:30.0," - + "logservice:60.0," - + "mail:30.0," - + "modules:60.0," - + "rdbms:60.0," - + "remote_socket:60.0," - + "search:10.0," - + "stubby:10.0", - // port is only used for the Java grpc server plugin in the Java8 gen 1, but needed. - "--port=0", - "--runtime_http_compression=true", - "--trusted_host=" + trustedPort, - "--use_env_vars_from_app_info=true", - "--use_jetty_http_proxy=true", - "--wait_for_daemon_request_threads=false", - "--log_json_to_var_log=true"); + "--trusted_host=" + trustedPort); } public static void main(String[] args) { diff --git a/runtime/nogaeapiswebapp/pom.xml b/runtime/nogaeapiswebapp/pom.xml index c39e8a82e..8cb7db23b 100644 --- a/runtime/nogaeapiswebapp/pom.xml +++ b/runtime/nogaeapiswebapp/pom.xml @@ -66,7 +66,7 @@ maven-compiler-plugin - 3.14.1 + 3.15.0 8 diff --git a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/AppInfoFactory.java b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/AppInfoFactory.java deleted file mode 100644 index 576bad4c5..000000000 --- a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/AppInfoFactory.java +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.apphosting.runtime.jetty; - -import static java.nio.charset.StandardCharsets.UTF_8; - -import com.esotericsoftware.yamlbeans.YamlReader; -import com.google.apphosting.base.protos.AppinfoPb; -import com.google.apphosting.utils.config.AppYaml; -import com.google.common.flogger.GoogleLogger; -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.NoSuchFileException; -import java.util.Map; -import org.jspecify.annotations.Nullable; - -/** Builds AppinfoPb.AppInfo from the given ServletEngineAdapter.Config and environment. */ -public class AppInfoFactory { - - private static final GoogleLogger logger = GoogleLogger.forEnclosingClass(); - - private static final String DEFAULT_CLOUD_PROJECT = "testapp"; - private static final String DEFAULT_GAE_APPLICATION = "s~testapp"; - private static final String DEFAULT_GAE_SERVICE = "default"; - private static final String DEFAULT_GAE_VERSION = "1.0"; - /** Path in the WAR layout to app.yaml */ - private static final String APP_YAML_PATH = "WEB-INF/appengine-generated/app.yaml"; - - private final String gaeVersion; - private final String googleCloudProject; - private final String gaeApplication; - private final String gaeService; - private final String gaeServiceVersion; - - public AppInfoFactory(Map env) { - String version = env.getOrDefault("GAE_VERSION", DEFAULT_GAE_VERSION); - String deploymentId = env.getOrDefault("GAE_DEPLOYMENT_ID", null); - gaeServiceVersion = (deploymentId != null) ? version + "." + deploymentId : version; - gaeService = env.getOrDefault("GAE_SERVICE", DEFAULT_GAE_SERVICE); - // Prepend service if it exists, otherwise do not prepend DEFAULT (go/app-engine-ids) - gaeVersion = - DEFAULT_GAE_SERVICE.equals(this.gaeService) - ? this.gaeServiceVersion - : this.gaeService + ":" + this.gaeServiceVersion; - googleCloudProject = env.getOrDefault("GOOGLE_CLOUD_PROJECT", DEFAULT_CLOUD_PROJECT); - gaeApplication = env.getOrDefault("GAE_APPLICATION", DEFAULT_GAE_APPLICATION); - } - - public String getGaeService() { - return gaeService; - } - - public String getGaeVersion() { - return gaeVersion; - } - - public String getGaeServiceVersion() { - return gaeServiceVersion; - } - - public String getGaeApplication() { - return gaeApplication; - } - - /** Creates a AppinfoPb.AppInfo object. */ - public AppinfoPb.AppInfo getAppInfoFromFile(String applicationRoot, String fixedApplicationPath) - throws IOException { - // App should be located under /base/data/home/apps/appId/versionID or in the optional - // fixedApplicationPath parameter. - String applicationPath = - (fixedApplicationPath == null) - ? applicationRoot + "/" + googleCloudProject + "/" + gaeServiceVersion - : fixedApplicationPath; - - if (!new File(applicationPath).exists()) { - throw new NoSuchFileException("Application does not exist under: " + applicationPath); - } - @Nullable String apiVersion = null; - File appYamlFile = new File(applicationPath, APP_YAML_PATH); - try { - YamlReader reader = new YamlReader(Files.newBufferedReader(appYamlFile.toPath(), UTF_8)); - Object apiVersionObj = ((Map) reader.read()).get("api_version"); - if (apiVersionObj != null) { - apiVersion = (String) apiVersionObj; - } - } catch (NoSuchFileException ex) { - logger.atInfo().log( - "Cannot configure App Engine APIs, because the generated app.yaml file " - + "does not exist: %s", - appYamlFile.getAbsolutePath()); - } - return getAppInfoWithApiVersion(apiVersion); - } - - public AppinfoPb.AppInfo getAppInfoFromAppYaml(AppYaml appYaml) throws IOException { - return getAppInfoWithApiVersion(appYaml.getApi_version()); - } - - public AppinfoPb.AppInfo getAppInfoWithApiVersion(@Nullable String apiVersion) { - final AppinfoPb.AppInfo.Builder appInfoBuilder = - AppinfoPb.AppInfo.newBuilder() - .setAppId(gaeApplication) - .setVersionId(gaeVersion) - .setRuntimeId("java8"); - - if (apiVersion != null) { - appInfoBuilder.setApiVersion(apiVersion); - } - - return appInfoBuilder.build(); - } -} diff --git a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/JettyServletEngineAdapter.java b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/JettyServletEngineAdapter.java index bf8f73504..8dbdaedcf 100644 --- a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/JettyServletEngineAdapter.java +++ b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/JettyServletEngineAdapter.java @@ -28,6 +28,7 @@ import com.google.apphosting.base.protos.RuntimePb.UPRequest; import com.google.apphosting.base.protos.RuntimePb.UPResponse; import com.google.apphosting.runtime.AppEngineConstants; +import com.google.apphosting.runtime.AppInfoFactory; import com.google.apphosting.runtime.AppVersion; import com.google.apphosting.runtime.LocalRpcContext; import com.google.apphosting.runtime.MutableUpResponse; @@ -161,40 +162,39 @@ public void run(Runnable runnable) { } boolean startJettyHttpProxy = false; - if (runtimeOptions.useJettyHttpProxy()) { - AppInfoFactory appInfoFactory; - AppVersionKey appVersionKey; - /* The init actions are not done in the constructor as they are not used when testing */ - try { - String appRoot = runtimeOptions.applicationRoot(); - String appPath = runtimeOptions.fixedApplicationPath(); - appInfoFactory = new AppInfoFactory(System.getenv()); - AppinfoPb.AppInfo appinfo = appInfoFactory.getAppInfoFromFile(appRoot, appPath); - // TODO Should we also call ApplyCloneSettings()? - LocalRpcContext context = new LocalRpcContext<>(EmptyMessage.class); - EvaluationRuntimeServerInterface evaluationRuntimeServerInterface = - Objects.requireNonNull(runtimeOptions.evaluationRuntimeServerInterface()); - evaluationRuntimeServerInterface.addAppVersion(context, appinfo); - context.getResponse(); - appVersionKey = AppVersionKey.fromAppInfo(appinfo); - } catch (Exception e) { - throw new IllegalStateException(e); - } - if (isHttpConnectorMode) { - logger.atInfo().log("Using HTTP_CONNECTOR_MODE to bypass RPC"); - server.insertHandler( - new JettyHttpHandler( - runtimeOptions, appVersionHandler.getAppVersion(), appVersionKey, appInfoFactory)); - JettyHttpProxy.insertHandlers(server, ignoreResponseSizeLimit); - server.addConnector(JettyHttpProxy.newConnector(server, runtimeOptions)); - } else { - server.setAttribute( - "com.google.apphosting.runtime.jetty.appYaml", - JettyServletEngineAdapter.getAppYaml(runtimeOptions)); - // Delay start of JettyHttpProxy until after the main server and application is started. - startJettyHttpProxy = true; - } + AppInfoFactory appInfoFactory; + AppVersionKey appVersionKey; + /* The init actions are not done in the constructor as they are not used when testing */ + try { + String appRoot = runtimeOptions.applicationRoot(); + String appPath = runtimeOptions.fixedApplicationPath(); + appInfoFactory = new AppInfoFactory(System.getenv()); + AppinfoPb.AppInfo appinfo = appInfoFactory.getAppInfoFromFile(appRoot, appPath); + // TODO Should we also call ApplyCloneSettings()? + LocalRpcContext context = new LocalRpcContext<>(EmptyMessage.class); + EvaluationRuntimeServerInterface evaluationRuntimeServerInterface = + Objects.requireNonNull(runtimeOptions.evaluationRuntimeServerInterface()); + evaluationRuntimeServerInterface.addAppVersion(context, appinfo); + context.getResponse(); + appVersionKey = AppVersionKey.fromAppInfo(appinfo); + } catch (Exception e) { + throw new IllegalStateException(e); + } + if (isHttpConnectorMode) { + logger.atInfo().log("Using HTTP_CONNECTOR_MODE to bypass RPC"); + server.insertHandler( + new JettyHttpHandler( + runtimeOptions, appVersionHandler.getAppVersion(), appVersionKey, appInfoFactory)); + JettyHttpProxy.insertHandlers(server, ignoreResponseSizeLimit); + server.addConnector(JettyHttpProxy.newConnector(server, runtimeOptions)); + } else { + server.setAttribute( + "com.google.apphosting.runtime.jetty.appYaml", + JettyServletEngineAdapter.getAppYaml(runtimeOptions)); + // Delay start of JettyHttpProxy until after the main server and application is started. + startJettyHttpProxy = true; } + try { server.start(); if (startJettyHttpProxy) { @@ -221,11 +221,6 @@ public void addAppVersion(AppVersion appVersion) { appVersionHandler.addAppVersion(appVersion); } - @Override - public void deleteAppVersion(AppVersion appVersion) { - appVersionHandler.removeAppVersion(appVersion.getKey()); - } - @Override public void setSessionStoreFactory(com.google.apphosting.runtime.SessionStoreFactory factory) { // No op with the new Jetty Session management. diff --git a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/http/JettyHttpHandler.java b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/http/JettyHttpHandler.java index 262affa18..07b6d44d4 100644 --- a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/http/JettyHttpHandler.java +++ b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/http/JettyHttpHandler.java @@ -25,26 +25,23 @@ import com.google.apphosting.base.protos.RuntimePb; import com.google.apphosting.runtime.ApiProxyImpl; import com.google.apphosting.runtime.AppEngineConstants; +import com.google.apphosting.runtime.AppInfoFactory; import com.google.apphosting.runtime.AppVersion; import com.google.apphosting.runtime.BackgroundRequestCoordinator; import com.google.apphosting.runtime.LocalRpcContext; import com.google.apphosting.runtime.RequestManager; -import com.google.apphosting.runtime.RequestRunner; import com.google.apphosting.runtime.RequestRunner.EagerRunner; import com.google.apphosting.runtime.ResponseAPIData; import com.google.apphosting.runtime.ServletEngineAdapter; import com.google.apphosting.runtime.anyrpc.AnyRpcServerContext; -import com.google.apphosting.runtime.jetty.AppInfoFactory; import com.google.common.flogger.GoogleLogger; import java.io.PrintWriter; import java.io.StringWriter; import java.time.Duration; import java.util.concurrent.TimeoutException; import org.eclipse.jetty.server.Handler; -import org.eclipse.jetty.server.HttpStream; import org.eclipse.jetty.server.Request; import org.eclipse.jetty.server.Response; -import org.eclipse.jetty.server.Server; import org.eclipse.jetty.util.Blocker; import org.eclipse.jetty.util.Callback; diff --git a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/http/JettyRequestAPIData.java b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/http/JettyRequestAPIData.java index 75d2e3a79..4b09f7275 100644 --- a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/http/JettyRequestAPIData.java +++ b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/http/JettyRequestAPIData.java @@ -55,9 +55,9 @@ import com.google.apphosting.base.protos.HttpPb; import com.google.apphosting.base.protos.RuntimePb; import com.google.apphosting.base.protos.TracePb; +import com.google.apphosting.runtime.AppInfoFactory; import com.google.apphosting.runtime.RequestAPIData; import com.google.apphosting.runtime.TraceContextHelper; -import com.google.apphosting.runtime.jetty.AppInfoFactory; import com.google.common.base.Strings; import com.google.common.flogger.GoogleLogger; import java.net.InetSocketAddress; diff --git a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/proxy/JettyHttpProxy.java b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/proxy/JettyHttpProxy.java index 5cfd8baba..08ee7d201 100644 --- a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/proxy/JettyHttpProxy.java +++ b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/proxy/JettyHttpProxy.java @@ -22,9 +22,9 @@ import com.google.apphosting.base.protos.RuntimePb.UPResponse; import com.google.apphosting.runtime.AppEngineConstants; import com.google.apphosting.runtime.LocalRpcContext; +import com.google.apphosting.runtime.AppInfoFactory; import com.google.apphosting.runtime.ServletEngineAdapter; import com.google.apphosting.runtime.anyrpc.EvaluationRuntimeServerInterface; -import com.google.apphosting.runtime.jetty.AppInfoFactory; import com.google.apphosting.runtime.jetty.AppVersionHandlerFactory; import com.google.common.base.Ascii; import com.google.common.base.Throwables; diff --git a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/proxy/UPRequestTranslator.java b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/proxy/UPRequestTranslator.java index 02c49757a..1e77cddbd 100644 --- a/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/proxy/UPRequestTranslator.java +++ b/runtime/runtime_impl_jetty12/src/main/java/com/google/apphosting/runtime/jetty/proxy/UPRequestTranslator.java @@ -56,8 +56,8 @@ import com.google.apphosting.base.protos.RuntimePb; import com.google.apphosting.base.protos.RuntimePb.UPRequest; import com.google.apphosting.base.protos.TracePb.TraceContextProto; +import com.google.apphosting.runtime.AppInfoFactory; import com.google.apphosting.runtime.TraceContextHelper; -import com.google.apphosting.runtime.jetty.AppInfoFactory; import com.google.common.base.Ascii; import com.google.common.base.Strings; import com.google.common.flogger.GoogleLogger; diff --git a/runtime/runtime_impl_jetty12/src/test/java/com/google/apphosting/runtime/jetty/AppInfoFactoryTest.java b/runtime/runtime_impl_jetty12/src/test/java/com/google/apphosting/runtime/jetty/AppInfoFactoryTest.java deleted file mode 100644 index 0afb0f0a5..000000000 --- a/runtime/runtime_impl_jetty12/src/test/java/com/google/apphosting/runtime/jetty/AppInfoFactoryTest.java +++ /dev/null @@ -1,243 +0,0 @@ -/* - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.apphosting.runtime.jetty; - -import static com.google.common.base.StandardSystemProperty.USER_DIR; -import static com.google.common.truth.Truth.assertThat; -import static java.nio.charset.StandardCharsets.UTF_8; -import static org.junit.Assert.assertThrows; - -import com.google.appengine.tools.development.resource.ResourceExtractor; -import com.google.apphosting.base.protos.AppinfoPb; -import com.google.apphosting.utils.config.AppYaml; -import com.google.common.collect.ImmutableMap; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStreamReader; -import java.nio.file.NoSuchFileException; -import java.nio.file.Path; -import java.nio.file.Paths; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -@RunWith(JUnit4.class) -public final class AppInfoFactoryTest { - private static final String PACKAGE_PATH = - AppInfoFactoryTest.class.getPackage().getName().replace('.', '/'); - private static final String PROJECT_RESOURCE_NAME = - String.format("%s/mytestproject", PACKAGE_PATH); - - @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); - - private String appRoot; - private String fixedAppDir; - - @Before - public void setUp() throws IOException { - Path projPath = Paths.get(temporaryFolder.newFolder(PROJECT_RESOURCE_NAME).getPath()); - appRoot = projPath.getParent().toString(); - fixedAppDir = Paths.get(projPath.toString(), "100.mydeployment").toString(); - ResourceExtractor.toFile(PROJECT_RESOURCE_NAME, projPath.toString()); - } - - @Test - public void getGaeService_nonDefault() throws Exception { - AppInfoFactory factory = - new AppInfoFactory(ImmutableMap.of("GAE_SERVICE", "mytestservice")); - assertThat(factory.getGaeService()).isEqualTo("mytestservice"); - } - - @Test - public void getGaeService_defaults() throws Exception { - AppInfoFactory factory = new AppInfoFactory(ImmutableMap.of()); - assertThat(factory.getGaeService()).isEqualTo("default"); - } - - @Test - public void getGaeVersion_nonDefaultWithDeploymentId() throws Exception { - AppInfoFactory factory = - new AppInfoFactory( - ImmutableMap.of( - "GAE_SERVICE", "mytestservice", - "GAE_DEPLOYMENT_ID", "mydeployment", - "GAE_VERSION", "100")); - assertThat(factory.getGaeVersion()).isEqualTo("mytestservice:100.mydeployment"); - } - - @Test - public void getGaeVersion_defaultWithDeploymentId() throws Exception { - AppInfoFactory factory = - new AppInfoFactory( - ImmutableMap.of( - "GAE_DEPLOYMENT_ID", "mydeployment", - "GAE_VERSION", "100")); - assertThat(factory.getGaeVersion()).isEqualTo("100.mydeployment"); - } - - @Test - public void getGaeVersion_defaultWithoutDeploymentId() throws Exception { - AppInfoFactory factory = new AppInfoFactory(ImmutableMap.of("GAE_VERSION", "100")); - assertThat(factory.getGaeVersion()).isEqualTo("100"); - } - - @Test - public void getGaeServiceVersion_withDeploymentId() throws Exception { - AppInfoFactory factory = - new AppInfoFactory( - ImmutableMap.of( - "GAE_DEPLOYMENT_ID", "mydeployment", - "GAE_VERSION", "100")); - assertThat(factory.getGaeVersion()).isEqualTo("100.mydeployment"); - } - - @Test - public void getGaeServiceVersion_withoutDeploymentId() throws Exception { - AppInfoFactory factory = new AppInfoFactory(ImmutableMap.of("GAE_VERSION", "100")); - assertThat(factory.getGaeVersion()).isEqualTo("100"); - } - - @Test - public void getGaeApplication_nonDefault() throws Exception { - AppInfoFactory factory = new AppInfoFactory(ImmutableMap.of("GAE_APPLICATION", "s~myapp")); - assertThat(factory.getGaeApplication()).isEqualTo("s~myapp"); - } - - @Test - public void getGaeApplication_defaults() throws Exception { - AppInfoFactory factory = new AppInfoFactory(ImmutableMap.of()); - assertThat(factory.getGaeApplication()).isEqualTo("s~testapp"); - } - - @Test - public void getAppInfo_fixedApplicationPath() throws Exception { - AppInfoFactory factory = - new AppInfoFactory( - ImmutableMap.of( - "GAE_SERVICE", "mytestservice", - "GAE_DEPLOYMENT_ID", "mydeployment", - "GAE_VERSION", "100", - "GAE_APPLICATION", "s~myapp")); - AppinfoPb.AppInfo appInfo = factory.getAppInfoFromFile(null, fixedAppDir); - - assertThat(appInfo.getAppId()).isEqualTo("s~myapp"); - assertThat(appInfo.getVersionId()).isEqualTo("mytestservice:100.mydeployment"); - assertThat(appInfo.getRuntimeId()).isEqualTo("java8"); - assertThat(appInfo.getApiVersion()).isEqualTo("200"); - } - - @Test - public void getAppInfo_appRoot() throws Exception { - AppInfoFactory factory = - new AppInfoFactory( - ImmutableMap.of( - "GAE_SERVICE", "mytestservice", - "GAE_DEPLOYMENT_ID", "mydeployment", - "GAE_VERSION", "100", - "GAE_APPLICATION", "s~myapp", - "GOOGLE_CLOUD_PROJECT", "mytestproject")); - AppinfoPb.AppInfo appInfo = factory.getAppInfoFromFile(appRoot, null); - - assertThat(appInfo.getAppId()).isEqualTo("s~myapp"); - assertThat(appInfo.getVersionId()).isEqualTo("mytestservice:100.mydeployment"); - assertThat(appInfo.getRuntimeId()).isEqualTo("java8"); - assertThat(appInfo.getApiVersion()).isEqualTo("200"); - } - - @Test - public void getAppInfo_noAppYaml() throws Exception { - AppInfoFactory factory = - new AppInfoFactory( - ImmutableMap.of( - "GAE_SERVICE", "mytestservice", - "GAE_DEPLOYMENT_ID", "mydeployment", - "GAE_VERSION", "100", - "GAE_APPLICATION", "s~myapp", - "GOOGLE_CLOUD_PROJECT", "bogusproject")); - AppinfoPb.AppInfo appInfo = - factory.getAppInfoFromFile( - null, - // We tell AppInfoFactory to look directly in the current working directory. There's no - // app.yaml there: - USER_DIR.value()); - - assertThat(appInfo.getAppId()).isEqualTo("s~myapp"); - assertThat(appInfo.getVersionId()).isEqualTo("mytestservice:100.mydeployment"); - assertThat(appInfo.getRuntimeId()).isEqualTo("java8"); - assertThat(appInfo.getApiVersion()).isEmpty(); - } - - @Test - public void getAppInfo_noDirectory() throws Exception { - AppInfoFactory factory = - new AppInfoFactory( - ImmutableMap.of( - "GAE_SERVICE", "mytestservice", - "GAE_DEPLOYMENT_ID", "mydeployment", - "GAE_VERSION", "100", - "GAE_APPLICATION", "s~myapp", - // This will make the AppInfoFactory hunt for a directory called bogusproject: - "GOOGLE_CLOUD_PROJECT", "bogusproject")); - - assertThrows(NoSuchFileException.class, () -> factory.getAppInfoFromFile(appRoot, null)); - } - - @Test - public void getAppInfo_givenAppYaml() throws Exception { - AppInfoFactory factory = - new AppInfoFactory( - ImmutableMap.of( - "GAE_SERVICE", "mytestservice", - "GAE_DEPLOYMENT_ID", "mydeployment", - "GAE_VERSION", "100", - "GAE_APPLICATION", "s~myapp", - "GOOGLE_CLOUD_PROJECT", "mytestproject")); - - File appYamlFile = new File(fixedAppDir + "/WEB-INF/appengine-generated/app.yaml"); - AppYaml appYaml = AppYaml.parse(new InputStreamReader(new FileInputStream(appYamlFile), UTF_8)); - - AppinfoPb.AppInfo appInfo = factory.getAppInfoFromAppYaml(appYaml); - - assertThat(appInfo.getAppId()).isEqualTo("s~myapp"); - assertThat(appInfo.getVersionId()).isEqualTo("mytestservice:100.mydeployment"); - assertThat(appInfo.getRuntimeId()).isEqualTo("java8"); - assertThat(appInfo.getApiVersion()).isEqualTo("200"); - } - - @Test - public void getAppInfo_givenVersion() throws Exception { - AppInfoFactory factory = - new AppInfoFactory( - ImmutableMap.of( - "GAE_SERVICE", "mytestservice", - "GAE_DEPLOYMENT_ID", "mydeployment", - "GAE_VERSION", "100", - "GAE_APPLICATION", "s~myapp", - "GOOGLE_CLOUD_PROJECT", "mytestproject")); - - AppinfoPb.AppInfo appInfo = factory.getAppInfoWithApiVersion("my_api_version"); - - assertThat(appInfo.getAppId()).isEqualTo("s~myapp"); - assertThat(appInfo.getVersionId()).isEqualTo("mytestservice:100.mydeployment"); - assertThat(appInfo.getRuntimeId()).isEqualTo("java8"); - assertThat(appInfo.getApiVersion()).isEqualTo("my_api_version"); - } -} diff --git a/runtime/runtime_impl_jetty12/src/test/java/com/google/apphosting/runtime/jetty/UPRequestTranslatorTest.java b/runtime/runtime_impl_jetty12/src/test/java/com/google/apphosting/runtime/jetty/UPRequestTranslatorTest.java index 1755b6a48..8d01e177b 100644 --- a/runtime/runtime_impl_jetty12/src/test/java/com/google/apphosting/runtime/jetty/UPRequestTranslatorTest.java +++ b/runtime/runtime_impl_jetty12/src/test/java/com/google/apphosting/runtime/jetty/UPRequestTranslatorTest.java @@ -32,6 +32,7 @@ import com.google.apphosting.base.protos.RuntimePb; import com.google.apphosting.base.protos.TraceId.TraceIdProto; import com.google.apphosting.base.protos.TracePb.TraceContextProto; +import com.google.apphosting.runtime.AppInfoFactory; import com.google.apphosting.runtime.jetty.proxy.UPRequestTranslator; import com.google.common.base.Ascii; import com.google.common.collect.ImmutableMap; diff --git a/runtime/runtime_impl_jetty121/src/main/java/com/google/apphosting/runtime/jetty/AppInfoFactory.java b/runtime/runtime_impl_jetty121/src/main/java/com/google/apphosting/runtime/jetty/AppInfoFactory.java deleted file mode 100644 index 7ceb0ef3e..000000000 --- a/runtime/runtime_impl_jetty121/src/main/java/com/google/apphosting/runtime/jetty/AppInfoFactory.java +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.apphosting.runtime.jetty; - -import static java.nio.charset.StandardCharsets.UTF_8; - -import com.esotericsoftware.yamlbeans.YamlReader; -import com.google.apphosting.base.protos.AppinfoPb; -import com.google.apphosting.utils.config.AppYaml; -import com.google.common.flogger.GoogleLogger; -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.NoSuchFileException; -import java.util.Map; -import org.jspecify.annotations.Nullable; - -/** Builds AppinfoPb.AppInfo from the given ServletEngineAdapter.Config and environment. */ -public class AppInfoFactory { - - private static final GoogleLogger logger = GoogleLogger.forEnclosingClass(); - - private static final String DEFAULT_CLOUD_PROJECT = "testapp"; - private static final String DEFAULT_GAE_APPLICATION = "s~testapp"; - private static final String DEFAULT_GAE_SERVICE = "default"; - private static final String DEFAULT_GAE_VERSION = "1.0"; - - /** Path in the WAR layout to app.yaml */ - private static final String APP_YAML_PATH = "WEB-INF/appengine-generated/app.yaml"; - - private final String gaeVersion; - private final String googleCloudProject; - private final String gaeApplication; - private final String gaeService; - private final String gaeServiceVersion; - - public AppInfoFactory(Map env) { - String version = env.getOrDefault("GAE_VERSION", DEFAULT_GAE_VERSION); - String deploymentId = env.getOrDefault("GAE_DEPLOYMENT_ID", null); - gaeServiceVersion = (deploymentId != null) ? version + "." + deploymentId : version; - gaeService = env.getOrDefault("GAE_SERVICE", DEFAULT_GAE_SERVICE); - // Prepend service if it exists, otherwise do not prepend DEFAULT (go/app-engine-ids) - gaeVersion = - DEFAULT_GAE_SERVICE.equals(this.gaeService) - ? this.gaeServiceVersion - : this.gaeService + ":" + this.gaeServiceVersion; - googleCloudProject = env.getOrDefault("GOOGLE_CLOUD_PROJECT", DEFAULT_CLOUD_PROJECT); - gaeApplication = env.getOrDefault("GAE_APPLICATION", DEFAULT_GAE_APPLICATION); - } - - public String getGaeService() { - return gaeService; - } - - public String getGaeVersion() { - return gaeVersion; - } - - public String getGaeServiceVersion() { - return gaeServiceVersion; - } - - public String getGaeApplication() { - return gaeApplication; - } - - /** Creates a AppinfoPb.AppInfo object. */ - public AppinfoPb.AppInfo getAppInfoFromFile(String applicationRoot, String fixedApplicationPath) - throws IOException { - // App should be located under /base/data/home/apps/appId/versionID or in the optional - // fixedApplicationPath parameter. - String applicationPath = - (fixedApplicationPath == null) - ? applicationRoot + "/" + googleCloudProject + "/" + gaeServiceVersion - : fixedApplicationPath; - - if (!new File(applicationPath).exists()) { - throw new NoSuchFileException("Application does not exist under: " + applicationPath); - } - @Nullable String apiVersion = null; - File appYamlFile = new File(applicationPath, APP_YAML_PATH); - try { - YamlReader reader = new YamlReader(Files.newBufferedReader(appYamlFile.toPath(), UTF_8)); - Object apiVersionObj = ((Map) reader.read()).get("api_version"); - if (apiVersionObj != null) { - apiVersion = (String) apiVersionObj; - } - } catch (NoSuchFileException ex) { - logger.atInfo().log( - "Cannot configure App Engine APIs, because the generated app.yaml file " - + "does not exist: %s", - appYamlFile.getAbsolutePath()); - } - return getAppInfoWithApiVersion(apiVersion); - } - - public AppinfoPb.AppInfo getAppInfoFromAppYaml(AppYaml appYaml) throws IOException { - return getAppInfoWithApiVersion(appYaml.getApi_version()); - } - - public AppinfoPb.AppInfo getAppInfoWithApiVersion(@Nullable String apiVersion) { - final AppinfoPb.AppInfo.Builder appInfoBuilder = - AppinfoPb.AppInfo.newBuilder() - .setAppId(gaeApplication) - .setVersionId(gaeVersion) - .setRuntimeId("java8"); - - if (apiVersion != null) { - appInfoBuilder.setApiVersion(apiVersion); - } - - return appInfoBuilder.build(); - } -} diff --git a/runtime/runtime_impl_jetty121/src/main/java/com/google/apphosting/runtime/jetty/JettyServletEngineAdapter.java b/runtime/runtime_impl_jetty121/src/main/java/com/google/apphosting/runtime/jetty/JettyServletEngineAdapter.java index 4c3e564f6..9bdefcfa8 100644 --- a/runtime/runtime_impl_jetty121/src/main/java/com/google/apphosting/runtime/jetty/JettyServletEngineAdapter.java +++ b/runtime/runtime_impl_jetty121/src/main/java/com/google/apphosting/runtime/jetty/JettyServletEngineAdapter.java @@ -28,6 +28,7 @@ import com.google.apphosting.base.protos.RuntimePb.UPRequest; import com.google.apphosting.base.protos.RuntimePb.UPResponse; import com.google.apphosting.runtime.AppEngineConstants; +import com.google.apphosting.runtime.AppInfoFactory; import com.google.apphosting.runtime.AppVersion; import com.google.apphosting.runtime.LocalRpcContext; import com.google.apphosting.runtime.MutableUpResponse; @@ -162,39 +163,40 @@ public void run(Runnable runnable) { } boolean startJettyHttpProxy = false; - if (runtimeOptions.useJettyHttpProxy()) { - AppInfoFactory appInfoFactory; - AppVersionKey appVersionKey; - /* The init actions are not done in the constructor as they are not used when testing */ - try { - String appRoot = runtimeOptions.applicationRoot(); - String appPath = runtimeOptions.fixedApplicationPath(); - appInfoFactory = new AppInfoFactory(System.getenv()); - AppinfoPb.AppInfo appinfo = appInfoFactory.getAppInfoFromFile(appRoot, appPath); - // TODO Should we also call ApplyCloneSettings()? - LocalRpcContext context = new LocalRpcContext<>(EmptyMessage.class); - EvaluationRuntimeServerInterface evaluationRuntimeServerInterface = - Objects.requireNonNull(runtimeOptions.evaluationRuntimeServerInterface()); - evaluationRuntimeServerInterface.addAppVersion(context, appinfo); - var unused = context.getResponse(); - appVersionKey = AppVersionKey.fromAppInfo(appinfo); - } catch (Exception e) { - throw new IllegalStateException(e); - } - if (isHttpConnectorMode) { - logger.atInfo().log("Using HTTP_CONNECTOR_MODE to bypass RPC"); - server.insertHandler( - new JettyHttpHandler( - runtimeOptions, appVersionHandler.getAppVersion(), appVersionKey, appInfoFactory)); - JettyHttpProxy.insertHandlers(server, ignoreResponseSizeLimit); - server.addConnector(JettyHttpProxy.newConnector(server, runtimeOptions)); - } else { - server.setAttribute( - "com.google.apphosting.runtime.jetty.appYaml", - JettyServletEngineAdapter.getAppYaml(runtimeOptions)); - // Delay start of JettyHttpProxy until after the main server and application is started. - startJettyHttpProxy = true; + AppInfoFactory appInfoFactory; + AppVersionKey appVersionKey; + /* The init actions are not done in the constructor as they are not used when testing */ + try { + String appRoot = runtimeOptions.applicationRoot(); + String appPath = runtimeOptions.fixedApplicationPath(); + appInfoFactory = new AppInfoFactory(System.getenv()); + AppinfoPb.AppInfo appinfo = appInfoFactory.getAppInfoFromFile(appRoot, appPath); + // TODO Should we also call ApplyCloneSettings()? + LocalRpcContext context = new LocalRpcContext<>(EmptyMessage.class); + EvaluationRuntimeServerInterface evaluationRuntimeServerInterface = + Objects.requireNonNull(runtimeOptions.evaluationRuntimeServerInterface()); + evaluationRuntimeServerInterface.addAppVersion(context, appinfo); + var unused = context.getResponse(); + appVersionKey = AppVersionKey.fromAppInfo(appinfo); + } catch (Exception e) { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); } + throw new IllegalStateException(e); + } + if (isHttpConnectorMode) { + logger.atInfo().log("Using HTTP_CONNECTOR_MODE to bypass RPC"); + server.insertHandler( + new JettyHttpHandler( + runtimeOptions, appVersionHandler.getAppVersion(), appVersionKey, appInfoFactory)); + JettyHttpProxy.insertHandlers(server, ignoreResponseSizeLimit); + server.addConnector(JettyHttpProxy.newConnector(server, runtimeOptions)); + } else { + server.setAttribute( + "com.google.apphosting.runtime.jetty.appYaml", + JettyServletEngineAdapter.getAppYaml(runtimeOptions)); + // Delay start of JettyHttpProxy until after the main server and application is started. + startJettyHttpProxy = true; } try { server.start(); @@ -222,11 +224,6 @@ public void addAppVersion(AppVersion appVersion) { appVersionHandler.addAppVersion(appVersion); } - @Override - public void deleteAppVersion(AppVersion appVersion) { - appVersionHandler.removeAppVersion(appVersion.getKey()); - } - @Override public void setSessionStoreFactory(com.google.apphosting.runtime.SessionStoreFactory factory) { // No op with the new Jetty Session management. diff --git a/runtime/runtime_impl_jetty121/src/main/java/com/google/apphosting/runtime/jetty/http/JettyHttpHandler.java b/runtime/runtime_impl_jetty121/src/main/java/com/google/apphosting/runtime/jetty/http/JettyHttpHandler.java index 9498b745e..7dcba3b70 100644 --- a/runtime/runtime_impl_jetty121/src/main/java/com/google/apphosting/runtime/jetty/http/JettyHttpHandler.java +++ b/runtime/runtime_impl_jetty121/src/main/java/com/google/apphosting/runtime/jetty/http/JettyHttpHandler.java @@ -32,9 +32,9 @@ import com.google.apphosting.runtime.RequestRunner; import com.google.apphosting.runtime.RequestRunner.EagerRunner; import com.google.apphosting.runtime.ResponseAPIData; +import com.google.apphosting.runtime.AppInfoFactory; import com.google.apphosting.runtime.ServletEngineAdapter; import com.google.apphosting.runtime.anyrpc.AnyRpcServerContext; -import com.google.apphosting.runtime.jetty.AppInfoFactory; import com.google.common.flogger.GoogleLogger; import java.io.PrintWriter; import java.io.StringWriter; @@ -170,7 +170,7 @@ private boolean dispatchRequest( requestManager.shutdownRequests(requestToken); return true; case BACKGROUND: - dispatchBackgroundRequest(request, response); + dispatchBackgroundRequest(request); return true; case OTHER: return dispatchServletRequest(request, response); @@ -194,7 +194,7 @@ private boolean dispatchServletRequest(JettyRequestAPIData request, JettyRespons } } - private void dispatchBackgroundRequest(JettyRequestAPIData request, JettyResponseAPIData response) + private void dispatchBackgroundRequest(JettyRequestAPIData request) throws InterruptedException, TimeoutException { String requestId = getBackgroundRequestId(request); // The interface of coordinator.waitForUserRunnable() requires us to provide the app code with a diff --git a/runtime/runtime_impl_jetty121/src/main/java/com/google/apphosting/runtime/jetty/http/JettyRequestAPIData.java b/runtime/runtime_impl_jetty121/src/main/java/com/google/apphosting/runtime/jetty/http/JettyRequestAPIData.java index 75d2e3a79..4b09f7275 100644 --- a/runtime/runtime_impl_jetty121/src/main/java/com/google/apphosting/runtime/jetty/http/JettyRequestAPIData.java +++ b/runtime/runtime_impl_jetty121/src/main/java/com/google/apphosting/runtime/jetty/http/JettyRequestAPIData.java @@ -55,9 +55,9 @@ import com.google.apphosting.base.protos.HttpPb; import com.google.apphosting.base.protos.RuntimePb; import com.google.apphosting.base.protos.TracePb; +import com.google.apphosting.runtime.AppInfoFactory; import com.google.apphosting.runtime.RequestAPIData; import com.google.apphosting.runtime.TraceContextHelper; -import com.google.apphosting.runtime.jetty.AppInfoFactory; import com.google.common.base.Strings; import com.google.common.flogger.GoogleLogger; import java.net.InetSocketAddress; diff --git a/runtime/runtime_impl_jetty121/src/main/java/com/google/apphosting/runtime/jetty/proxy/JettyHttpProxy.java b/runtime/runtime_impl_jetty121/src/main/java/com/google/apphosting/runtime/jetty/proxy/JettyHttpProxy.java index e52a75525..5d35902ca 100644 --- a/runtime/runtime_impl_jetty121/src/main/java/com/google/apphosting/runtime/jetty/proxy/JettyHttpProxy.java +++ b/runtime/runtime_impl_jetty121/src/main/java/com/google/apphosting/runtime/jetty/proxy/JettyHttpProxy.java @@ -22,9 +22,9 @@ import com.google.apphosting.base.protos.RuntimePb.UPResponse; import com.google.apphosting.runtime.AppEngineConstants; import com.google.apphosting.runtime.LocalRpcContext; +import com.google.apphosting.runtime.AppInfoFactory; import com.google.apphosting.runtime.ServletEngineAdapter; import com.google.apphosting.runtime.anyrpc.EvaluationRuntimeServerInterface; -import com.google.apphosting.runtime.jetty.AppInfoFactory; import com.google.apphosting.runtime.jetty.AppVersionHandlerFactory; import com.google.common.base.Ascii; import com.google.common.base.Throwables; diff --git a/runtime/runtime_impl_jetty121/src/main/java/com/google/apphosting/runtime/jetty/proxy/UPRequestTranslator.java b/runtime/runtime_impl_jetty121/src/main/java/com/google/apphosting/runtime/jetty/proxy/UPRequestTranslator.java index 02c49757a..1e77cddbd 100644 --- a/runtime/runtime_impl_jetty121/src/main/java/com/google/apphosting/runtime/jetty/proxy/UPRequestTranslator.java +++ b/runtime/runtime_impl_jetty121/src/main/java/com/google/apphosting/runtime/jetty/proxy/UPRequestTranslator.java @@ -56,8 +56,8 @@ import com.google.apphosting.base.protos.RuntimePb; import com.google.apphosting.base.protos.RuntimePb.UPRequest; import com.google.apphosting.base.protos.TracePb.TraceContextProto; +import com.google.apphosting.runtime.AppInfoFactory; import com.google.apphosting.runtime.TraceContextHelper; -import com.google.apphosting.runtime.jetty.AppInfoFactory; import com.google.common.base.Ascii; import com.google.common.base.Strings; import com.google.common.flogger.GoogleLogger; diff --git a/runtime/runtime_impl_jetty121/src/test/java/com/google/apphosting/runtime/grpc/FakeApiProxyImplFactory.java b/runtime/runtime_impl_jetty121/src/test/java/com/google/apphosting/runtime/grpc/FakeApiProxyImplFactory.java index 18ed563fe..d6ec4057f 100644 --- a/runtime/runtime_impl_jetty121/src/test/java/com/google/apphosting/runtime/grpc/FakeApiProxyImplFactory.java +++ b/runtime/runtime_impl_jetty121/src/test/java/com/google/apphosting/runtime/grpc/FakeApiProxyImplFactory.java @@ -41,10 +41,7 @@ public static ApiProxyImpl newApiProxyImpl(APIHostClientInterface apiHostClient) return ApiProxyImpl.builder() .setApiHost(apiHostClient) .setDeadlineOracle( - new ApiDeadlineOracle.Builder() - .initDeadlineMap(30, "", 30, "") - .initOfflineDeadlineMap(30, "", 30, "") - .build()) + new ApiDeadlineOracle.Builder().initDeadlineMap().build()) .setByteCountBeforeFlushing(8192) .setMaxLogFlushTime(Duration.ofSeconds(5)) .build(); diff --git a/runtime/runtime_impl_jetty121/src/test/java/com/google/apphosting/runtime/jetty/AppInfoFactoryTest.java b/runtime/runtime_impl_jetty121/src/test/java/com/google/apphosting/runtime/jetty/AppInfoFactoryTest.java deleted file mode 100644 index 0f142f7d6..000000000 --- a/runtime/runtime_impl_jetty121/src/test/java/com/google/apphosting/runtime/jetty/AppInfoFactoryTest.java +++ /dev/null @@ -1,242 +0,0 @@ -/* - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.apphosting.runtime.jetty; - -import static com.google.common.base.StandardSystemProperty.USER_DIR; -import static com.google.common.truth.Truth.assertThat; -import static java.nio.charset.StandardCharsets.UTF_8; -import static org.junit.Assert.assertThrows; - -import com.google.appengine.tools.development.resource.ResourceExtractor; -import com.google.apphosting.base.protos.AppinfoPb; -import com.google.apphosting.utils.config.AppYaml; -import com.google.common.collect.ImmutableMap; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStreamReader; -import java.nio.file.NoSuchFileException; -import java.nio.file.Path; -import java.nio.file.Paths; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -@RunWith(JUnit4.class) -public final class AppInfoFactoryTest { - private static final String PACKAGE_PATH = - AppInfoFactoryTest.class.getPackage().getName().replace('.', '/'); - private static final String PROJECT_RESOURCE_NAME = - String.format("%s/mytestproject", PACKAGE_PATH); - - @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); - - private String appRoot; - private String fixedAppDir; - - @Before - public void setUp() throws IOException { - Path projPath = Paths.get(temporaryFolder.newFolder(PROJECT_RESOURCE_NAME).getPath()); - appRoot = projPath.getParent().toString(); - fixedAppDir = Paths.get(projPath.toString(), "100.mydeployment").toString(); - ResourceExtractor.toFile(PROJECT_RESOURCE_NAME, projPath.toString()); - } - - @Test - public void getGaeService_nonDefault() throws Exception { - AppInfoFactory factory = new AppInfoFactory(ImmutableMap.of("GAE_SERVICE", "mytestservice")); - assertThat(factory.getGaeService()).isEqualTo("mytestservice"); - } - - @Test - public void getGaeService_defaults() throws Exception { - AppInfoFactory factory = new AppInfoFactory(ImmutableMap.of()); - assertThat(factory.getGaeService()).isEqualTo("default"); - } - - @Test - public void getGaeVersion_nonDefaultWithDeploymentId() throws Exception { - AppInfoFactory factory = - new AppInfoFactory( - ImmutableMap.of( - "GAE_SERVICE", "mytestservice", - "GAE_DEPLOYMENT_ID", "mydeployment", - "GAE_VERSION", "100")); - assertThat(factory.getGaeVersion()).isEqualTo("mytestservice:100.mydeployment"); - } - - @Test - public void getGaeVersion_defaultWithDeploymentId() throws Exception { - AppInfoFactory factory = - new AppInfoFactory( - ImmutableMap.of( - "GAE_DEPLOYMENT_ID", "mydeployment", - "GAE_VERSION", "100")); - assertThat(factory.getGaeVersion()).isEqualTo("100.mydeployment"); - } - - @Test - public void getGaeVersion_defaultWithoutDeploymentId() throws Exception { - AppInfoFactory factory = new AppInfoFactory(ImmutableMap.of("GAE_VERSION", "100")); - assertThat(factory.getGaeVersion()).isEqualTo("100"); - } - - @Test - public void getGaeServiceVersion_withDeploymentId() throws Exception { - AppInfoFactory factory = - new AppInfoFactory( - ImmutableMap.of( - "GAE_DEPLOYMENT_ID", "mydeployment", - "GAE_VERSION", "100")); - assertThat(factory.getGaeVersion()).isEqualTo("100.mydeployment"); - } - - @Test - public void getGaeServiceVersion_withoutDeploymentId() throws Exception { - AppInfoFactory factory = new AppInfoFactory(ImmutableMap.of("GAE_VERSION", "100")); - assertThat(factory.getGaeVersion()).isEqualTo("100"); - } - - @Test - public void getGaeApplication_nonDefault() throws Exception { - AppInfoFactory factory = new AppInfoFactory(ImmutableMap.of("GAE_APPLICATION", "s~myapp")); - assertThat(factory.getGaeApplication()).isEqualTo("s~myapp"); - } - - @Test - public void getGaeApplication_defaults() throws Exception { - AppInfoFactory factory = new AppInfoFactory(ImmutableMap.of()); - assertThat(factory.getGaeApplication()).isEqualTo("s~testapp"); - } - - @Test - public void getAppInfo_fixedApplicationPath() throws Exception { - AppInfoFactory factory = - new AppInfoFactory( - ImmutableMap.of( - "GAE_SERVICE", "mytestservice", - "GAE_DEPLOYMENT_ID", "mydeployment", - "GAE_VERSION", "100", - "GAE_APPLICATION", "s~myapp")); - AppinfoPb.AppInfo appInfo = factory.getAppInfoFromFile(null, fixedAppDir); - - assertThat(appInfo.getAppId()).isEqualTo("s~myapp"); - assertThat(appInfo.getVersionId()).isEqualTo("mytestservice:100.mydeployment"); - assertThat(appInfo.getRuntimeId()).isEqualTo("java8"); - assertThat(appInfo.getApiVersion()).isEqualTo("200"); - } - - @Test - public void getAppInfo_appRoot() throws Exception { - AppInfoFactory factory = - new AppInfoFactory( - ImmutableMap.of( - "GAE_SERVICE", "mytestservice", - "GAE_DEPLOYMENT_ID", "mydeployment", - "GAE_VERSION", "100", - "GAE_APPLICATION", "s~myapp", - "GOOGLE_CLOUD_PROJECT", "mytestproject")); - AppinfoPb.AppInfo appInfo = factory.getAppInfoFromFile(appRoot, null); - - assertThat(appInfo.getAppId()).isEqualTo("s~myapp"); - assertThat(appInfo.getVersionId()).isEqualTo("mytestservice:100.mydeployment"); - assertThat(appInfo.getRuntimeId()).isEqualTo("java8"); - assertThat(appInfo.getApiVersion()).isEqualTo("200"); - } - - @Test - public void getAppInfo_noAppYaml() throws Exception { - AppInfoFactory factory = - new AppInfoFactory( - ImmutableMap.of( - "GAE_SERVICE", "mytestservice", - "GAE_DEPLOYMENT_ID", "mydeployment", - "GAE_VERSION", "100", - "GAE_APPLICATION", "s~myapp", - "GOOGLE_CLOUD_PROJECT", "bogusproject")); - AppinfoPb.AppInfo appInfo = - factory.getAppInfoFromFile( - null, - // We tell AppInfoFactory to look directly in the current working directory. There's no - // app.yaml there: - USER_DIR.value()); - - assertThat(appInfo.getAppId()).isEqualTo("s~myapp"); - assertThat(appInfo.getVersionId()).isEqualTo("mytestservice:100.mydeployment"); - assertThat(appInfo.getRuntimeId()).isEqualTo("java8"); - assertThat(appInfo.getApiVersion()).isEmpty(); - } - - @Test - public void getAppInfo_noDirectory() throws Exception { - AppInfoFactory factory = - new AppInfoFactory( - ImmutableMap.of( - "GAE_SERVICE", "mytestservice", - "GAE_DEPLOYMENT_ID", "mydeployment", - "GAE_VERSION", "100", - "GAE_APPLICATION", "s~myapp", - // This will make the AppInfoFactory hunt for a directory called bogusproject: - "GOOGLE_CLOUD_PROJECT", "bogusproject")); - - assertThrows(NoSuchFileException.class, () -> factory.getAppInfoFromFile(appRoot, null)); - } - - @Test - public void getAppInfo_givenAppYaml() throws Exception { - AppInfoFactory factory = - new AppInfoFactory( - ImmutableMap.of( - "GAE_SERVICE", "mytestservice", - "GAE_DEPLOYMENT_ID", "mydeployment", - "GAE_VERSION", "100", - "GAE_APPLICATION", "s~myapp", - "GOOGLE_CLOUD_PROJECT", "mytestproject")); - - File appYamlFile = new File(fixedAppDir + "/WEB-INF/appengine-generated/app.yaml"); - AppYaml appYaml = AppYaml.parse(new InputStreamReader(new FileInputStream(appYamlFile), UTF_8)); - - AppinfoPb.AppInfo appInfo = factory.getAppInfoFromAppYaml(appYaml); - - assertThat(appInfo.getAppId()).isEqualTo("s~myapp"); - assertThat(appInfo.getVersionId()).isEqualTo("mytestservice:100.mydeployment"); - assertThat(appInfo.getRuntimeId()).isEqualTo("java8"); - assertThat(appInfo.getApiVersion()).isEqualTo("200"); - } - - @Test - public void getAppInfo_givenVersion() throws Exception { - AppInfoFactory factory = - new AppInfoFactory( - ImmutableMap.of( - "GAE_SERVICE", "mytestservice", - "GAE_DEPLOYMENT_ID", "mydeployment", - "GAE_VERSION", "100", - "GAE_APPLICATION", "s~myapp", - "GOOGLE_CLOUD_PROJECT", "mytestproject")); - - AppinfoPb.AppInfo appInfo = factory.getAppInfoWithApiVersion("my_api_version"); - - assertThat(appInfo.getAppId()).isEqualTo("s~myapp"); - assertThat(appInfo.getVersionId()).isEqualTo("mytestservice:100.mydeployment"); - assertThat(appInfo.getRuntimeId()).isEqualTo("java8"); - assertThat(appInfo.getApiVersion()).isEqualTo("my_api_version"); - } -} diff --git a/runtime/runtime_impl_jetty121/src/test/java/com/google/apphosting/runtime/jetty/UPRequestTranslatorTest.java b/runtime/runtime_impl_jetty121/src/test/java/com/google/apphosting/runtime/jetty/UPRequestTranslatorTest.java index 85df696fa..4abefab54 100644 --- a/runtime/runtime_impl_jetty121/src/test/java/com/google/apphosting/runtime/jetty/UPRequestTranslatorTest.java +++ b/runtime/runtime_impl_jetty121/src/test/java/com/google/apphosting/runtime/jetty/UPRequestTranslatorTest.java @@ -32,24 +32,19 @@ import com.google.apphosting.base.protos.RuntimePb; import com.google.apphosting.base.protos.TraceId.TraceIdProto; import com.google.apphosting.base.protos.TracePb.TraceContextProto; +import com.google.apphosting.runtime.AppInfoFactory; import com.google.apphosting.runtime.jetty.proxy.UPRequestTranslator; import com.google.common.base.Ascii; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.protobuf.ExtensionRegistry; import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.OutputStream; import java.net.SocketAddress; import java.net.URI; import java.net.URISyntaxException; import java.nio.ByteBuffer; import java.util.HashMap; import java.util.Map; -import javax.servlet.ReadListener; -import javax.servlet.ServletInputStream; -import javax.servlet.ServletOutputStream; -import javax.servlet.WriteListener; import javax.servlet.http.HttpServletResponse; import org.eclipse.jetty.http.HttpFields; import org.eclipse.jetty.http.HttpURI; @@ -68,6 +63,7 @@ import org.mockito.stubbing.Answer; @RunWith(JUnit4.class) +@SuppressWarnings("GoogleHttpHeaderConstants") public final class UPRequestTranslatorTest { private static final String X_APPENGINE_HTTPS = "X-AppEngine-Https"; private static final String X_APPENGINE_USER_IP = "X-AppEngine-User-IP"; @@ -463,46 +459,4 @@ private static Request mockServletRequest( return httpRequest; } - private static ServletInputStream emptyInputStream() { - return new ServletInputStream() { - @Override - public int read() { - return -1; - } - - @Override - public void setReadListener(ReadListener listener) { - throw new UnsupportedOperationException(); - } - - @Override - public boolean isReady() { - return true; - } - - @Override - public boolean isFinished() { - return true; - } - }; - } - - private static ServletOutputStream copyingOutputStream(OutputStream out) { - return new ServletOutputStream() { - @Override - public void write(int b) throws IOException { - out.write(b); - } - - @Override - public void setWriteListener(WriteListener listener) { - throw new UnsupportedOperationException(); - } - - @Override - public boolean isReady() { - return true; - } - }; - } } diff --git a/runtime/runtime_impl_jetty9/src/main/java/com/google/apphosting/runtime/jetty9/JettyHttpHandler.java b/runtime/runtime_impl_jetty9/src/main/java/com/google/apphosting/runtime/jetty9/JettyHttpHandler.java index e8f682364..3a6ba66be 100644 --- a/runtime/runtime_impl_jetty9/src/main/java/com/google/apphosting/runtime/jetty9/JettyHttpHandler.java +++ b/runtime/runtime_impl_jetty9/src/main/java/com/google/apphosting/runtime/jetty9/JettyHttpHandler.java @@ -26,11 +26,11 @@ import com.google.apphosting.base.protos.RuntimePb; import com.google.apphosting.runtime.ApiProxyImpl; import com.google.apphosting.runtime.AppEngineConstants; +import com.google.apphosting.runtime.AppInfoFactory; import com.google.apphosting.runtime.AppVersion; import com.google.apphosting.runtime.BackgroundRequestCoordinator; import com.google.apphosting.runtime.LocalRpcContext; import com.google.apphosting.runtime.RequestManager; -import com.google.apphosting.runtime.RequestRunner; import com.google.apphosting.runtime.RequestRunner.EagerRunner; import com.google.apphosting.runtime.ResponseAPIData; import com.google.apphosting.runtime.ServletEngineAdapter; @@ -41,19 +41,17 @@ import java.io.StringWriter; import java.time.Duration; import java.util.concurrent.TimeoutException; -import org.jspecify.annotations.Nullable; import javax.servlet.ServletException; import javax.servlet.UnavailableException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.eclipse.jetty.http.BadMessageException; import org.eclipse.jetty.http.HttpStatus; -import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.HttpChannel; import org.eclipse.jetty.server.Request; -import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.server.handler.HandlerWrapper; +import org.jspecify.annotations.Nullable; /** * This class replicates the behaviour of the {@link RequestRunner} for Requests which do not come diff --git a/runtime/runtime_impl_jetty9/src/main/java/com/google/apphosting/runtime/jetty9/JettyHttpProxy.java b/runtime/runtime_impl_jetty9/src/main/java/com/google/apphosting/runtime/jetty9/JettyHttpProxy.java index 9a2998a94..dbc7ef02e 100644 --- a/runtime/runtime_impl_jetty9/src/main/java/com/google/apphosting/runtime/jetty9/JettyHttpProxy.java +++ b/runtime/runtime_impl_jetty9/src/main/java/com/google/apphosting/runtime/jetty9/JettyHttpProxy.java @@ -24,6 +24,7 @@ import com.google.apphosting.base.protos.RuntimePb.UPRequest; import com.google.apphosting.base.protos.RuntimePb.UPResponse; import com.google.apphosting.runtime.LocalRpcContext; +import com.google.apphosting.runtime.AppInfoFactory; import com.google.apphosting.runtime.ServletEngineAdapter; import com.google.apphosting.runtime.anyrpc.EvaluationRuntimeServerInterface; import com.google.common.base.Ascii; diff --git a/runtime/runtime_impl_jetty9/src/main/java/com/google/apphosting/runtime/jetty9/JettyRequestAPIData.java b/runtime/runtime_impl_jetty9/src/main/java/com/google/apphosting/runtime/jetty9/JettyRequestAPIData.java index e4168f8c2..f12726bed 100644 --- a/runtime/runtime_impl_jetty9/src/main/java/com/google/apphosting/runtime/jetty9/JettyRequestAPIData.java +++ b/runtime/runtime_impl_jetty9/src/main/java/com/google/apphosting/runtime/jetty9/JettyRequestAPIData.java @@ -57,6 +57,7 @@ import com.google.apphosting.base.protos.RuntimePb; import com.google.apphosting.base.protos.TracePb; import com.google.apphosting.base.protos.TracePb.TraceContextProto; +import com.google.apphosting.runtime.AppInfoFactory; import com.google.apphosting.runtime.RequestAPIData; import com.google.apphosting.runtime.TraceContextHelper; import com.google.common.base.Strings; diff --git a/runtime/runtime_impl_jetty9/src/main/java/com/google/apphosting/runtime/jetty9/JettyServletEngineAdapter.java b/runtime/runtime_impl_jetty9/src/main/java/com/google/apphosting/runtime/jetty9/JettyServletEngineAdapter.java index 629a8c19d..ae76d47d8 100644 --- a/runtime/runtime_impl_jetty9/src/main/java/com/google/apphosting/runtime/jetty9/JettyServletEngineAdapter.java +++ b/runtime/runtime_impl_jetty9/src/main/java/com/google/apphosting/runtime/jetty9/JettyServletEngineAdapter.java @@ -27,6 +27,7 @@ import com.google.apphosting.base.protos.RuntimePb.UPRequest; import com.google.apphosting.base.protos.RuntimePb.UPResponse; import com.google.apphosting.runtime.AppVersion; +import com.google.apphosting.runtime.AppInfoFactory; import com.google.apphosting.runtime.LocalRpcContext; import com.google.apphosting.runtime.MutableUpResponse; import com.google.apphosting.runtime.ServletEngineAdapter; @@ -127,8 +128,7 @@ public void start(String serverInfo, ServletEngineAdapter.Config runtimeOptions) try { boolean startJettyHttpProxy = false; - if (runtimeOptions.useJettyHttpProxy()) { - AppVersionKey appVersionKey; + AppVersionKey appVersionKey; /* The init actions are not done in the constructor as they are not used when testing */ String appRoot = runtimeOptions.applicationRoot(); String appPath = runtimeOptions.fixedApplicationPath(); @@ -157,7 +157,6 @@ public void start(String serverInfo, ServletEngineAdapter.Config runtimeOptions) // Delay start of JettyHttpProxy until after the main server and application is started. startJettyHttpProxy = true; } - } ClassLoader oldContextClassLoader = Thread.currentThread().getContextClassLoader(); Thread.currentThread() @@ -192,11 +191,6 @@ public void addAppVersion(AppVersion appVersion) { appVersionHandlerMap.addAppVersion(appVersion); } - @Override - public void deleteAppVersion(AppVersion appVersion) { - appVersionHandlerMap.removeAppVersion(appVersion.getKey()); - } - @Override public void setSessionStoreFactory(SessionStoreFactory factory) { // No op with the new Jetty Session management. diff --git a/runtime/runtime_impl_jetty9/src/main/java/com/google/apphosting/runtime/jetty9/UPRequestTranslator.java b/runtime/runtime_impl_jetty9/src/main/java/com/google/apphosting/runtime/jetty9/UPRequestTranslator.java index 4673926df..62a0038cb 100644 --- a/runtime/runtime_impl_jetty9/src/main/java/com/google/apphosting/runtime/jetty9/UPRequestTranslator.java +++ b/runtime/runtime_impl_jetty9/src/main/java/com/google/apphosting/runtime/jetty9/UPRequestTranslator.java @@ -55,6 +55,7 @@ import com.google.apphosting.base.protos.RuntimePb; import com.google.apphosting.base.protos.RuntimePb.UPRequest; import com.google.apphosting.base.protos.TracePb.TraceContextProto; +import com.google.apphosting.runtime.AppInfoFactory; import com.google.apphosting.runtime.TraceContextHelper; import com.google.common.base.Ascii; import com.google.common.base.Strings; diff --git a/runtime/runtime_impl_jetty9/src/test/java/com/google/apphosting/runtime/CloneControllerImplTest.java b/runtime/runtime_impl_jetty9/src/test/java/com/google/apphosting/runtime/CloneControllerImplTest.java deleted file mode 100644 index 2e71f1d03..000000000 --- a/runtime/runtime_impl_jetty9/src/test/java/com/google/apphosting/runtime/CloneControllerImplTest.java +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.apphosting.runtime; - -import static com.google.common.truth.Truth.assertThat; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import com.google.apphosting.base.protos.ClonePb.PerformanceData; -import com.google.apphosting.base.protos.ModelClonePb.PerformanceDataRequest; -import com.google.apphosting.runtime.test.MockAnyRpcServerContext; -import com.google.common.io.ByteStreams; -import java.io.InputStream; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.time.Duration; -import java.util.Arrays; -import java.util.Optional; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -/** - * Unit tests for the CloneControllerImpl class. - * - */ -@RunWith(JUnit4.class) -public class CloneControllerImplTest { - private static final double API_DEADLINE = 10.0; - private static final Duration RPC_DEADLINE = Duration.ofSeconds(3); - private static final long MAX_RUNTIME_LOG_PER_REQUEST = 3000L * 1024L; - private static final long SOFT_DEADLINE_DELAY = 750; - private static final int HSPERFDATA_SIZE = 32768; - private static final int FAKE_HSPERFDATA_SIZE = 100; - private static final long CYCLES_PER_SECOND = 2333414000L; - - private final JavaRuntime javaRuntime = mock(JavaRuntime.class); - private final AppVersion appVersion = mock(AppVersion.class); - - @Before - public void setUp() { - when(javaRuntime.findAppVersion("app1", "1.1")).thenReturn(appVersion); - } - - @Test - public void testPerformanceDataDisabled() { - CloneControllerImpl cloneController = createCloneController(null); - MockAnyRpcServerContext rpc = createRpc(); - cloneController.getPerformanceData(rpc, PerformanceDataRequest.getDefaultInstance()); - PerformanceData data = (PerformanceData) rpc.assertSuccess(); - assertThat(data.getEntriesCount()).isEqualTo(0); - } - - @Test - public void testPerformanceDataEnabled() { - byte[] fakePerformanceData = new byte[FAKE_HSPERFDATA_SIZE]; - for (int i = 0; i < FAKE_HSPERFDATA_SIZE; ++i) { - fakePerformanceData[i] = (byte) i; - } - ByteBuffer perfData = ByteBuffer.wrap(fakePerformanceData).order(ByteOrder.LITTLE_ENDIAN); - CloneControllerImpl cloneController = createCloneController(perfData); - MockAnyRpcServerContext rpc = createRpc(); - cloneController.getPerformanceData(rpc, PerformanceDataRequest.getDefaultInstance()); - PerformanceData data = (PerformanceData) rpc.assertSuccess(); - assertThat(data.getEntriesCount()).isEqualTo(1); - PerformanceData.Entry entry = data.getEntries(0); - assertThat(entry.getFormat()).isEqualTo(PerformanceData.Format.JAVA_HOTSPOT_HSPERFDATA); - byte[] payload = entry.getPayload().toByteArray(); - assertThat(payload).isEqualTo(fakePerformanceData); - } - - @Test - public void testPerformanceDataEnabledAgainstRealPerfData() throws Exception { - // This is just a file that I fished out of /tmp/hsperfdata_emcmanus at some point. - // It's from a Maven process. - byte[] perfData; - try (InputStream in = getClass().getResourceAsStream("hsperf.data")) { - perfData = ByteStreams.toByteArray(in); - } - ByteBuffer realPerformanceData = ByteBuffer.wrap(perfData); - CloneControllerImpl cloneController = createCloneController(realPerformanceData); - MockAnyRpcServerContext rpc = createRpc(); - cloneController.getPerformanceData(rpc, PerformanceDataRequest.getDefaultInstance()); - PerformanceData data = (PerformanceData) rpc.assertSuccess(); - assertThat(data.getEntriesCount()).isEqualTo(1); - PerformanceData.Entry entry = data.getEntries(0); - assertThat(entry.getFormat()).isEqualTo(PerformanceData.Format.JAVA_HOTSPOT_HSPERFDATA); - byte[] payload = entry.getPayload().toByteArray(); - assertThat(payload).hasLength(HSPERFDATA_SIZE); - // Check the magic header (4 bytes). - assertThat(Arrays.copyOf(payload, 4)) - .isEqualTo(new byte[] {(byte) 0xCA, (byte) 0xFE, (byte) 0xC0, (byte) 0xC0}); - } - - private CloneControllerImpl createCloneController(ByteBuffer hotspotPerformanceData) { - return new CloneControllerImpl( - javaRuntime.new CloneControllerImplCallback(), - new ApiDeadlineOracle.Builder() - .initDeadlineMap(API_DEADLINE, "", 0.0, "") - .initOfflineDeadlineMap(API_DEADLINE, "", 0.0, "") - .build(), - createRequestManager(true, false), - hotspotPerformanceData); - } - - private RequestManager createRequestManager( - boolean terminateClones, boolean interruptOnSoftDeadline) { - return RequestManager.builder() - .setSoftDeadlineDelay(SOFT_DEADLINE_DELAY) - .setRuntimeLogSink(Optional.of(new RuntimeLogSink(MAX_RUNTIME_LOG_PER_REQUEST))) - .setApiProxyImpl(ApiProxyImpl.builder().build()) - .setMaxOutstandingApiRpcs(10) - .setThreadStopTerminatesClone(terminateClones) - .setInterruptFirstOnSoftDeadline(interruptOnSoftDeadline) - .setCyclesPerSecond(CYCLES_PER_SECOND) - .setWaitForDaemonRequestThreads(true) - .build(); - } - - private MockAnyRpcServerContext createRpc() { - return new MockAnyRpcServerContext(RPC_DEADLINE); - } -} diff --git a/runtime/runtime_impl_jetty9/src/test/java/com/google/apphosting/runtime/JavaRuntimeParamsTest.java b/runtime/runtime_impl_jetty9/src/test/java/com/google/apphosting/runtime/JavaRuntimeParamsTest.java index 5013e780e..dfc4995b8 100644 --- a/runtime/runtime_impl_jetty9/src/test/java/com/google/apphosting/runtime/JavaRuntimeParamsTest.java +++ b/runtime/runtime_impl_jetty9/src/test/java/com/google/apphosting/runtime/JavaRuntimeParamsTest.java @@ -44,18 +44,7 @@ public class JavaRuntimeParamsTest { @Rule public Expect expect = Expect.create(); - @Test - public void testSomeDefaults() { - JavaRuntimeParams params = JavaRuntimeParams.parseArgs(); - assertThat(params.getAppengineReleaseName()).isEqualTo("unknown"); - } - @Test - public void testEqualsSeparator() { - String[] args = {"--application_root=ROOT"}; - JavaRuntimeParams params = JavaRuntimeParams.parseArgs(args); - assertThat(params.getApplicationRoot()).isEqualTo("ROOT"); - } @Test public void testBooleanDefaultTrue() { @@ -149,8 +138,9 @@ public void testBooleanParamsArityIsOne() { if (param != null) { Class fieldType = field.getType(); if (fieldType == boolean.class || fieldType == Boolean.class) { - assertWithMessage("Boolean param " + param.names()[0] + " must have arity=1") - .that(param.arity()).isEqualTo(1); + assertWithMessage("Boolean param %s must have arity=1", param.names()[0]) + .that(param.arity()) + .isEqualTo(1); } } } @@ -164,8 +154,9 @@ public void testBooleanParamsDoNotStartWithNo() { Class fieldType = field.getType(); if (fieldType == boolean.class || fieldType == Boolean.class) { for (String name : param.names()) { - assertWithMessage("Boolean param " + name + " cannot start with '--no'") - .that(name).doesNotMatch("--no.*"); + assertWithMessage("Boolean param %s cannot start with '--no'", name) + .that(name) + .doesNotMatch("--no.*"); } } } @@ -221,88 +212,40 @@ public void testExpandBooleanParamsUnknownParams() { assertThat(params).containsExactly("--unknown1", "--nounknown2").inOrder(); } - @Test - public void testApplicationRootDefault() { - JavaRuntimeParams params = JavaRuntimeParams.parseArgs(); - assertThat(params.getApplicationRoot()).isEqualTo("appdata"); - } - - @Test - public void testApplicationRoot() { - String[] args = {"--application_root=abc"}; - JavaRuntimeParams params = JavaRuntimeParams.parseArgs(args); - assertThat(params.getApplicationRoot()).isEqualTo("abc"); - } - @Test - public void testFixedApplicationPath() { - String[] args = {"--fixed_application_path=abc"}; - JavaRuntimeParams params = JavaRuntimeParams.parseArgs(args); - assertThat(params.getFixedApplicationPath()).isEqualTo("abc"); - } @Test public void testUnknownArgumentsAllowed() { String[] args = {"--xyz=abc"}; - JavaRuntimeParams.parseArgs(args); + var unused = JavaRuntimeParams.parseArgs(args); } @Test public void testDefaults() { JavaRuntimeParams params = JavaRuntimeParams.parseArgs(); - assertThat(params.getApplicationRoot()).isEqualTo("appdata"); - assertThat(params.getPort()).isEqualTo(0); assertThat(params.getTrustedHost()).isEmpty(); - assertThat(params.getJavaSoftDeadlineMs()).isEqualTo(600); - assertThat(params.getApiCallDeadline()).isEqualTo(5.0); - assertThat(params.getMaxApiCallDeadline()).isEqualTo(10.0); - assertThat(params.getApiCallDeadlineMap()).isEmpty(); - assertThat(params.getMaxApiCallDeadlineMap()).isEmpty(); - assertThat(params.getOfflineApiCallDeadline()).isEqualTo(5.0); - assertThat(params.getMaxOfflineApiCallDeadline()).isEqualTo(10.0); // Skipped entropyString. - assertThat(params.getAppengineReleaseName()).isEqualTo("unknown"); assertThat(params.getLogJettyExceptionsToAppLogs()).isTrue(); - assertThat(params.getExternalDatacenterName()).isNull(); assertThat(params.getCloneMaxOutstandingApiRpcs()).isEqualTo(100); assertThat(params.getThreadStopTerminatesClone()).isTrue(); // Skipped deprecated params. assertThat(params.getByteCountBeforeFlushing()).isEqualTo(100 * 1024L); assertThat(params.getMaxLogLineSize()).isEqualTo(16 * 1024); assertThat(params.getMaxLogFlushSeconds()).isEqualTo(60); - assertThat(params.getRuntimeHttpCompression()).isFalse(); assertThat(params.getMaxRuntimeLogPerRequest()).isEqualTo(3000L * 1024L); - assertThat(params.getEnableGaeCloudSqlJdbcConnectivity()).isFalse(); - assertThat(params.getForceReadaheadOnCloudsqlSocket()).isFalse(); // Skipped deprecated params. - assertThat(params.getInterruptThreadsFirstOnSoftDeadline()).isFalse(); assertThat(params.getEnableHotspotPerformanceMetrics()).isFalse(); assertThat(params.getEnableCloudCpuProfiler()).isFalse(); assertThat(params.getEnableCloudHeapProfiler()).isFalse(); assertThat(params.getUrlfetchDeriveResponseMessage()).isTrue(); - assertThat(params.getCyclesPerSecond()).isEqualTo(0L); - assertThat(params.getMailFilenamePreventsInlining()).isFalse(); - assertThat(params.getMailSupportExtendedAttachmentEncodings()).isFalse(); - assertThat(params.getWaitForDaemonRequestThreads()).isTrue(); assertThat(params.getPollForNetwork()).isFalse(); - assertThat(params.getDefaultToNativeUrlStreamHandler()).isFalse(); assertThat(params.getForceUrlfetchUrlStreamHandler()).isFalse(); - assertThat(params.getEnableSynchronizedAppLogsWriter()).isTrue(); - assertThat(params.getUseEnvVarsFromAppInfo()).isFalse(); assertThat(params.getFixedApplicationPath()).isNull(); assertThat(params.getDisableApiCallLogging()).isFalse(); - assertThat(params.getLogJsonToVarLog()).isFalse(); - } - - @Test - public void testAllowDuplicateParams() { - String[] args = {"--application_root=1", "--application_root=2"}; - JavaRuntimeParams params = JavaRuntimeParams.parseArgs(args); - assertThat(params.getApplicationRoot()).isEqualTo("2"); } @Test public void testGetUnknownParams() { - String[] args = {"--unknown1=xyz", "--application_root=abc", "--unknown2=xyz"}; + String[] args = {"--unknown1=xyz", "--trusted_host=abc", "--unknown2=xyz"}; JavaRuntimeParams params = JavaRuntimeParams.parseArgs(args); assertThat(params.getUnknownParams()) .containsExactly("--unknown1=xyz", "--unknown2=xyz") diff --git a/runtime/runtime_impl_jetty9/src/test/java/com/google/apphosting/runtime/RequestManagerTest.java b/runtime/runtime_impl_jetty9/src/test/java/com/google/apphosting/runtime/RequestManagerTest.java index a6b1931df..a60b30cda 100644 --- a/runtime/runtime_impl_jetty9/src/test/java/com/google/apphosting/runtime/RequestManagerTest.java +++ b/runtime/runtime_impl_jetty9/src/test/java/com/google/apphosting/runtime/RequestManagerTest.java @@ -121,11 +121,9 @@ private RequestManager.Builder requestManagerBuilder() { .setSoftDeadlineDelay(SOFT_DEADLINE_DELAY) .setRuntimeLogSink(Optional.of(logSink)) .setApiProxyImpl(ApiProxyImpl.builder().setApiHost(mockApiHost).build()) - .setMaxOutstandingApiRpcs(10) + .setMaxOutstandingApiRpcs(100) .setCyclesPerSecond(CYCLES_PER_SECOND) - .setWaitForDaemonRequestThreads(true) - .setThreadStopTerminatesClone(true) - .setInterruptFirstOnSoftDeadline(false); + .setThreadStopTerminatesClone(true); } private RequestManager createRequestManager() { diff --git a/runtime/runtime_impl_jetty9/src/test/java/com/google/apphosting/runtime/RequestRunnerTest.java b/runtime/runtime_impl_jetty9/src/test/java/com/google/apphosting/runtime/RequestRunnerTest.java index d30eda960..d87defbea 100644 --- a/runtime/runtime_impl_jetty9/src/test/java/com/google/apphosting/runtime/RequestRunnerTest.java +++ b/runtime/runtime_impl_jetty9/src/test/java/com/google/apphosting/runtime/RequestRunnerTest.java @@ -153,7 +153,6 @@ public void serviceRequest(UPRequest upRequest, MutableUpResponse upResponse) .setUpResponse(upResponse) .setRequestManager(requestManager) .setCoordinator(coordinator) - .setCompressResponse(true) .setUpRequestHandler(servletEngine) .build(); @@ -205,7 +204,6 @@ public void serviceRequest(UPRequest upRequest, MutableUpResponse upResponse) .setUpResponse(upResponse) .setRequestManager(requestManager) .setCoordinator(coordinator) - .setCompressResponse(true) .setUpRequestHandler(servletEngine) .build(); @@ -323,7 +321,6 @@ public void run_backgroundRequest() .setUpResponse(upResponse) .setRequestManager(requestManager) .setCoordinator(coordinator) - .setCompressResponse(true) .setUpRequestHandler(servletEngine) .build(); diff --git a/runtime/runtime_impl_jetty9/src/test/java/com/google/apphosting/runtime/grpc/FakeApiProxyImplFactory.java b/runtime/runtime_impl_jetty9/src/test/java/com/google/apphosting/runtime/grpc/FakeApiProxyImplFactory.java index 18ed563fe..d6ec4057f 100644 --- a/runtime/runtime_impl_jetty9/src/test/java/com/google/apphosting/runtime/grpc/FakeApiProxyImplFactory.java +++ b/runtime/runtime_impl_jetty9/src/test/java/com/google/apphosting/runtime/grpc/FakeApiProxyImplFactory.java @@ -41,10 +41,7 @@ public static ApiProxyImpl newApiProxyImpl(APIHostClientInterface apiHostClient) return ApiProxyImpl.builder() .setApiHost(apiHostClient) .setDeadlineOracle( - new ApiDeadlineOracle.Builder() - .initDeadlineMap(30, "", 30, "") - .initOfflineDeadlineMap(30, "", 30, "") - .build()) + new ApiDeadlineOracle.Builder().initDeadlineMap().build()) .setByteCountBeforeFlushing(8192) .setMaxLogFlushTime(Duration.ofSeconds(5)) .build(); diff --git a/runtime/runtime_impl_jetty9/src/test/java/com/google/apphosting/runtime/jetty9/UPRequestTranslatorTest.java b/runtime/runtime_impl_jetty9/src/test/java/com/google/apphosting/runtime/jetty9/UPRequestTranslatorTest.java index 2d90b76b8..98d636988 100644 --- a/runtime/runtime_impl_jetty9/src/test/java/com/google/apphosting/runtime/jetty9/UPRequestTranslatorTest.java +++ b/runtime/runtime_impl_jetty9/src/test/java/com/google/apphosting/runtime/jetty9/UPRequestTranslatorTest.java @@ -32,6 +32,7 @@ import com.google.apphosting.base.protos.RuntimePb; import com.google.apphosting.base.protos.TraceId.TraceIdProto; import com.google.apphosting.base.protos.TracePb.TraceContextProto; +import com.google.apphosting.runtime.AppInfoFactory; import com.google.common.base.Ascii; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -58,6 +59,7 @@ import org.mockito.stubbing.Answer; @RunWith(JUnit4.class) +@SuppressWarnings("GoogleHttpHeaderConstants") public final class UPRequestTranslatorTest { private static final String X_APPENGINE_HTTPS = "X-AppEngine-Https"; private static final String X_APPENGINE_USER_IP = "X-AppEngine-User-IP"; diff --git a/runtime/test/src/test/java/com/google/apphosting/runtime/jetty9/SpringBootTest.java b/runtime/test/src/test/java/com/google/apphosting/runtime/jetty9/SpringBootTest.java index 0a3518601..52ef3e8d6 100644 --- a/runtime/test/src/test/java/com/google/apphosting/runtime/jetty9/SpringBootTest.java +++ b/runtime/test/src/test/java/com/google/apphosting/runtime/jetty9/SpringBootTest.java @@ -28,7 +28,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Locale; -import java.util.stream.Collectors; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -124,7 +123,7 @@ private RuntimeContext runtimeContext() throws IOException, Inte private static List readOutput(InputStream inputStream) throws IOException { try (BufferedReader output = new BufferedReader(new InputStreamReader(inputStream))) { - return output.lines().map(l -> l + "\n").collect(Collectors.toList()); + return output.lines().map(l -> l + "\n").toList(); } } diff --git a/runtime/test/src/test/java/com/google/apphosting/runtime/tests/GuestBookTest.java b/runtime/test/src/test/java/com/google/apphosting/runtime/tests/GuestBookTest.java index 34294cf69..0d1051242 100644 --- a/runtime/test/src/test/java/com/google/apphosting/runtime/tests/GuestBookTest.java +++ b/runtime/test/src/test/java/com/google/apphosting/runtime/tests/GuestBookTest.java @@ -31,7 +31,6 @@ import java.net.http.HttpResponse; import java.util.List; import java.util.Locale; -import java.util.stream.Collectors; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -100,7 +99,7 @@ private RuntimeContext runtimeContext() throws IOException, InterruptedExcept private static List readOutput(InputStream inputStream) throws IOException { try (BufferedReader output = new BufferedReader(new InputStreamReader(inputStream))) { - return output.lines().map(l -> l + "\n").collect(Collectors.toList()); + return output.lines().map(l -> l + "\n").toList(); } } diff --git a/runtime/testapps/src/main/java/com/google/apphosting/loadtesting/allinone/MainServlet.java b/runtime/testapps/src/main/java/com/google/apphosting/loadtesting/allinone/MainServlet.java index 4926dd42f..8949deb52 100644 --- a/runtime/testapps/src/main/java/com/google/apphosting/loadtesting/allinone/MainServlet.java +++ b/runtime/testapps/src/main/java/com/google/apphosting/loadtesting/allinone/MainServlet.java @@ -440,12 +440,7 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws S */ private void performMathMs(int ms, PrintWriter w) { emitf(w, "Burning cpu for %d ms", ms); - runRepeatedly(ms, new Runnable() { - @Override - public void run() { - performMath(random.nextBoolean()); - } - }); + runRepeatedly(ms, () -> performMath(random.nextBoolean())); logger.info("Cpu burned"); } diff --git a/runtime/testapps/src/main/java/com/google/apphosting/loadtesting/allinone/ee10/MainServlet.java b/runtime/testapps/src/main/java/com/google/apphosting/loadtesting/allinone/ee10/MainServlet.java index cdc63f922..35aa96305 100644 --- a/runtime/testapps/src/main/java/com/google/apphosting/loadtesting/allinone/ee10/MainServlet.java +++ b/runtime/testapps/src/main/java/com/google/apphosting/loadtesting/allinone/ee10/MainServlet.java @@ -440,12 +440,7 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws S */ private void performMathMs(int ms, PrintWriter w) { emitf(w, "Burning cpu for %d ms", ms); - runRepeatedly(ms, new Runnable() { - @Override - public void run() { - performMath(random.nextBoolean()); - } - }); + runRepeatedly(ms, () -> performMath(random.nextBoolean())); logger.info("Cpu burned"); } diff --git a/runtime/util/src/main/java/com/google/apphosting/runtime/ApplicationEnvironment.java b/runtime/util/src/main/java/com/google/apphosting/runtime/ApplicationEnvironment.java index aa7dba4e1..f06a7f56c 100644 --- a/runtime/util/src/main/java/com/google/apphosting/runtime/ApplicationEnvironment.java +++ b/runtime/util/src/main/java/com/google/apphosting/runtime/ApplicationEnvironment.java @@ -70,14 +70,7 @@ public class ApplicationEnvironment { */ @AutoValue public abstract static class RuntimeConfiguration { - public static final RuntimeConfiguration DEFAULT_FOR_TEST = builder() - .setCloudSqlJdbcConnectivityEnabled(false) - .setUseGoogleConnectorJ(false) - .build(); - - public abstract boolean getCloudSqlJdbcConnectivityEnabled(); - - public abstract boolean getUseGoogleConnectorJ(); + public static final RuntimeConfiguration DEFAULT_FOR_TEST = builder().build(); public static Builder builder() { return new AutoValue_ApplicationEnvironment_RuntimeConfiguration.Builder(); @@ -88,11 +81,6 @@ public static Builder builder() { /** Builder for RuntimeConfiguration. */ @AutoValue.Builder public abstract static class Builder { - public abstract Builder setCloudSqlJdbcConnectivityEnabled( - boolean cloudSqlJdbcConnectivityEnabled); - - public abstract Builder setUseGoogleConnectorJ(boolean useGoogleConnectorJ); - public abstract RuntimeConfiguration build(); } } @@ -171,6 +159,6 @@ public void setUseGoogleConnectorJ(@Nullable Boolean useGoogleConnectorJ) { public boolean getUseGoogleConnectorJ() { return Optional.ofNullable(useGoogleConnectorJ) - .orElse(runtimeConfiguration.getUseGoogleConnectorJ()); + .orElse(true); } } diff --git a/runtime/util/src/main/java/com/google/apphosting/runtime/NullSandboxPlugin.java b/runtime/util/src/main/java/com/google/apphosting/runtime/NullSandboxPlugin.java index 21a17bc4d..dd63a598d 100644 --- a/runtime/util/src/main/java/com/google/apphosting/runtime/NullSandboxPlugin.java +++ b/runtime/util/src/main/java/com/google/apphosting/runtime/NullSandboxPlugin.java @@ -131,13 +131,10 @@ protected ClassLoader doCreateApplicationClassLoader( // methods were copied from UserClassLoaderFactory.createClassLoader. URL[] prebundledUrls = getClassPathUtils().getPrebundledUrls(); userUrls = append(userUrls, prebundledUrls); - // Add the j-connector class path at the front. Also copied from - // UserClassLoaderFactory.createClassLoader. - if (environment.getRuntimeConfiguration().getCloudSqlJdbcConnectivityEnabled() - && environment.getUseGoogleConnectorJ()) { - URL[] urls = getClassPathUtils().getConnectorJUrls(); - userUrls = append(urls, userUrls); - } + // Add the j-connector class path at the front. Also copied from + // UserClassLoaderFactory.createClassLoader. + URL[] urls = getClassPathUtils().getConnectorJUrls(); + userUrls = append(urls, userUrls); boolean alwaysScanClassDirs = "true".equalsIgnoreCase( environment.getSystemProperties().get(ALWAYS_SCAN_CLASS_DIRS_PROPERTY)); return new ApplicationClassLoader( diff --git a/runtime_shared/src/main/java/com/google/apphosting/api/ApiProxy.java b/runtime_shared/src/main/java/com/google/apphosting/api/ApiProxy.java index 601501bfc..b51e350b0 100644 --- a/runtime_shared/src/main/java/com/google/apphosting/api/ApiProxy.java +++ b/runtime_shared/src/main/java/com/google/apphosting/api/ApiProxy.java @@ -706,6 +706,7 @@ public int hashCode() { * this interface. However, callers should not currently assume * that all RPCs will. */ + @SuppressWarnings("ShouldNotSubclass") public interface ApiResultFuture extends Future { /** * Returns the amount of CPU time consumed across any backend diff --git a/shared_sdk/src/main/java/com/google/appengine/tools/development/Clock.java b/shared_sdk/src/main/java/com/google/appengine/tools/development/Clock.java index fa364db38..68a5cb775 100644 --- a/shared_sdk/src/main/java/com/google/appengine/tools/development/Clock.java +++ b/shared_sdk/src/main/java/com/google/appengine/tools/development/Clock.java @@ -31,10 +31,5 @@ public interface Clock { * The Clock instance used by local services if no override is * provided via {@link ApiProxyLocal#setClock(Clock)} */ - Clock DEFAULT = new Clock() { - @Override - public long getCurrentTime() { - return System.currentTimeMillis(); - } - }; + Clock DEFAULT = System::currentTimeMillis; } diff --git a/utils/src/main/java/com/google/apphosting/utils/config/BackendsXml.java b/utils/src/main/java/com/google/apphosting/utils/config/BackendsXml.java index 15223bd94..b3f9b6386 100644 --- a/utils/src/main/java/com/google/apphosting/utils/config/BackendsXml.java +++ b/utils/src/main/java/com/google/apphosting/utils/config/BackendsXml.java @@ -69,22 +69,24 @@ public String toYaml() { if (!backends.isEmpty()) { builder.append("backends:\n"); for (BackendsXml.Entry entry : backends) { - builder.append("- name: " + entry.getName() + "\n"); + builder.append("- name: ").append(entry.getName()).append("\n"); if (entry.getInstances() != null) { - builder.append(" instances: " + entry.getInstances() + "\n"); + builder.append(" instances: ").append(entry.getInstances()).append("\n"); } if (entry.getInstanceClass() != null) { - builder.append(" class: " + entry.getInstanceClass() + "\n"); + builder.append(" class: ").append(entry.getInstanceClass()).append("\n"); } if (entry.getMaxConcurrentRequests() != null) { - builder.append(" max_concurrent_requests: " + entry.getMaxConcurrentRequests() + "\n"); + builder.append(" max_concurrent_requests: ").append(entry.getMaxConcurrentRequests()) + .append("\n"); } List options = new ArrayList(); for (BackendsXml.Option option : entry.getOptions()) { options.add(option.getYamlValue()); } if (!options.isEmpty()) { - builder.append(" options: " + Joiner.on(", ").useForNull("null").join(options) + "\n"); + builder.append(" options: ").append(Joiner.on(", ").useForNull("null").join(options)) + .append("\n"); } } } @@ -239,19 +241,19 @@ public String toString() { builder.append("Backend: "); builder.append(name); if (instances != null) { - builder.append(", instances = " + instances); + builder.append(", instances = ").append(instances); } if (instanceClass != null) { - builder.append(", instanceClass = " + instanceClass); + builder.append(", instanceClass = ").append(instanceClass); } if (maxConcurrentRequests != null) { - builder.append(", maxConcurrentRequests = " + maxConcurrentRequests); + builder.append(", maxConcurrentRequests = ").append(maxConcurrentRequests); } if (options != null) { - builder.append(", options = " + options); + builder.append(", options = ").append(options); } if (state != null) { - builder.append(", state = " + state); + builder.append(", state = ").append(state); } return builder.toString(); } diff --git a/utils/src/main/java/com/google/apphosting/utils/config/CronXml.java b/utils/src/main/java/com/google/apphosting/utils/config/CronXml.java index 31ad56b95..ef36d60d6 100644 --- a/utils/src/main/java/com/google/apphosting/utils/config/CronXml.java +++ b/utils/src/main/java/com/google/apphosting/utils/config/CronXml.java @@ -200,13 +200,14 @@ public String toYaml() { StringBuilder builder = new StringBuilder("cron:\n"); for (Entry ent : entries) { // description may contain YAML special characters. - builder.append("- description: '" + ent.getDescription().replace("'", "''") + "'\n"); - builder.append(" url: " + ent.getUrl() + "\n"); - builder.append(" schedule: " + ent.getSchedule() + "\n"); - builder.append(" timezone: " + ent.getTimezone() + "\n"); + builder.append("- description: '").append(ent.getDescription().replace("'", "''")) + .append("'\n"); + builder.append(" url: ").append(ent.getUrl()).append("\n"); + builder.append(" schedule: ").append(ent.getSchedule()).append("\n"); + builder.append(" timezone: ").append(ent.getTimezone()).append("\n"); String target = ent.getTarget(); if (target != null) { - builder.append(" target: " + target + "\n"); + builder.append(" target: ").append(target).append("\n"); } RetryParametersXml retryParameters = ent.getRetryParameters(); if (retryParameters != null) { diff --git a/utils/src/main/java/com/google/apphosting/utils/config/DosXml.java b/utils/src/main/java/com/google/apphosting/utils/config/DosXml.java index 4d367189f..badd0951e 100644 --- a/utils/src/main/java/com/google/apphosting/utils/config/DosXml.java +++ b/utils/src/main/java/com/google/apphosting/utils/config/DosXml.java @@ -138,9 +138,9 @@ public void validateLastEntry() { public String toYaml() { StringBuilder builder = new StringBuilder("blacklist:\n"); for (BlacklistEntry ent : blacklistEntries) { - builder.append("- subnet: " + ent.getSubnet() + "\n"); + builder.append("- subnet: ").append(ent.getSubnet()).append("\n"); if (!ent.getDescription().equals("")) { - builder.append(" description: " + ent.getDescription() + "\n"); + builder.append(" description: ").append(ent.getDescription()).append("\n"); } } return builder.toString(); diff --git a/utils/src/main/java/com/google/apphosting/utils/config/IndexesXml.java b/utils/src/main/java/com/google/apphosting/utils/config/IndexesXml.java index 3d292f5be..327fd006d 100644 --- a/utils/src/main/java/com/google/apphosting/utils/config/IndexesXml.java +++ b/utils/src/main/java/com/google/apphosting/utils/config/IndexesXml.java @@ -129,21 +129,21 @@ public String toYaml() { */ private String toLocalStyleYaml(){ StringBuilder builder = new StringBuilder(50 * (1 + properties.size())); - builder.append("- kind: \"" + kind + "\"\n"); + builder.append("- kind: \"").append(kind).append("\"\n"); if (Boolean.TRUE.equals(ancestors)) { builder.append(" ancestor: yes\n"); } if (!properties.isEmpty()) { builder.append(" properties:\n"); for (PropertySort prop : properties) { - builder.append(" - name: \"" + prop.getPropertyName() + "\"\n"); + builder.append(" - name: \"").append(prop.getPropertyName()).append("\"\n"); if (prop.getDirection() != null) { - builder.append(" direction: " + prop.getDirection() + "\n"); + builder.append(" direction: ").append(prop.getDirection()).append("\n"); } if (prop.getMode() != null) { - builder.append(" mode: " + prop.getMode() + "\n"); + builder.append(" mode: ").append(prop.getMode()).append("\n"); } } } @@ -162,7 +162,7 @@ private String toLocalStyleYaml(){ private String toServerStyleYaml() { StringBuilder builder = new StringBuilder(50 * (1 + properties.size())); builder.append("- ").append(IndexYamlReader.INDEX_TAG).append("\n"); - builder.append(" kind: " + kind + "\n"); + builder.append(" kind: ").append(kind).append("\n"); if (Boolean.TRUE.equals(ancestors)) { builder.append(" ancestor: yes\n"); } @@ -186,7 +186,7 @@ private String toServerStyleYaml() { } builder.append(" "); - builder.append("name: " + prop.getPropertyName()); + builder.append("name: ").append(prop.getPropertyName()); builder.append("}\n"); } } @@ -197,16 +197,17 @@ public String toXmlString() { StringBuilder builder = new StringBuilder(100 * (1 + properties.size())); String ancestorAttribute = ancestors == null ? "" : String.format(" ancestor=\"%s\"", ancestors); - builder.append("\n"); + builder.append("\n"); for (PropertySort prop : properties) { - builder.append(" \n"); diff --git a/utils/src/main/java/com/google/apphosting/utils/config/QueueXml.java b/utils/src/main/java/com/google/apphosting/utils/config/QueueXml.java index 7741d0490..4d65f596a 100644 --- a/utils/src/main/java/com/google/apphosting/utils/config/QueueXml.java +++ b/utils/src/main/java/com/google/apphosting/utils/config/QueueXml.java @@ -600,59 +600,68 @@ public String getTotalStorageLimit() { public String toYaml() { StringBuilder builder = new StringBuilder(); if (getTotalStorageLimit().length() > 0) { - builder.append("total_storage_limit: " + getTotalStorageLimit() + "\n\n"); + builder.append("total_storage_limit: ").append(getTotalStorageLimit()).append("\n\n"); } builder.append("queue:\n"); for (Entry ent : getEntries()) { - builder.append("- name: " + ent.getName() + "\n"); + builder.append("- name: ").append(ent.getName()).append("\n"); Double rate = ent.getRate(); if (rate != null) { - builder.append( - " rate: " + rate + '/' + ent.getRateUnit().getIdent() + "\n"); + builder + .append(" rate: ") + .append(rate) + .append('/') + .append(ent.getRateUnit().getIdent()) + .append("\n"); } Integer bucketSize = ent.getBucketSize(); if (bucketSize != null) { - builder.append(" bucket_size: " + bucketSize + "\n"); + builder.append(" bucket_size: ").append(bucketSize).append("\n"); } Integer maxConcurrentRequests = ent.getMaxConcurrentRequests(); if (maxConcurrentRequests != null) { - builder.append(" max_concurrent_requests: " + maxConcurrentRequests + "\n"); + builder.append(" max_concurrent_requests: ").append(maxConcurrentRequests).append("\n"); } RetryParameters retryParameters = ent.getRetryParameters(); if (retryParameters != null) { builder.append(" retry_parameters:\n"); if (retryParameters.getRetryLimit() != null) { - builder.append(" task_retry_limit: " + retryParameters.getRetryLimit() + "\n"); + builder.append(" task_retry_limit: ").append(retryParameters.getRetryLimit()) + .append("\n"); } if (retryParameters.getAgeLimitSec() != null) { - builder.append(" task_age_limit: " + retryParameters.getAgeLimitSec() + "s\n"); + builder.append(" task_age_limit: ").append(retryParameters.getAgeLimitSec()) + .append("s\n"); } if (retryParameters.getMinBackoffSec() != null) { - builder.append(" min_backoff_seconds: " + retryParameters.getMinBackoffSec() + "\n"); + builder.append(" min_backoff_seconds: ").append(retryParameters.getMinBackoffSec()) + .append("\n"); } if (retryParameters.getMaxBackoffSec() != null) { - builder.append(" max_backoff_seconds: " + retryParameters.getMaxBackoffSec() + "\n"); + builder.append(" max_backoff_seconds: ").append(retryParameters.getMaxBackoffSec()) + .append("\n"); } if (retryParameters.getMaxDoublings() != null) { - builder.append(" max_doublings: " + retryParameters.getMaxDoublings() + "\n"); + builder.append(" max_doublings: ").append(retryParameters.getMaxDoublings()) + .append("\n"); } } String target = ent.getTarget(); if (target != null) { - builder.append(" target: " + target + "\n"); + builder.append(" target: ").append(target).append("\n"); } String mode = ent.getMode(); if (mode != null) { - builder.append(" mode: " + mode + "\n"); + builder.append(" mode: ").append(mode).append("\n"); } List acl = ent.getAcl(); if (acl != null) { builder.append(" acl:\n"); for (AclEntry aclEntry : acl) { if (aclEntry.getUserEmail() != null) { - builder.append(" - user_email: " + aclEntry.getUserEmail() + "\n"); + builder.append(" - user_email: ").append(aclEntry.getUserEmail()).append("\n"); } else if (aclEntry.getWriterEmail() != null) { - builder.append(" - writer_email: " + aclEntry.getWriterEmail() + "\n"); + builder.append(" - writer_email: ").append(aclEntry.getWriterEmail()).append("\n"); } } } diff --git a/utils/src/main/java/com/google/apphosting/utils/config/RetryParametersXml.java b/utils/src/main/java/com/google/apphosting/utils/config/RetryParametersXml.java index a9a385b35..01a8c3b22 100644 --- a/utils/src/main/java/com/google/apphosting/utils/config/RetryParametersXml.java +++ b/utils/src/main/java/com/google/apphosting/utils/config/RetryParametersXml.java @@ -55,14 +55,14 @@ public enum AgeLimitUnit { } static AgeLimitUnit valueOf(char unit) { - switch (unit) { - case 's' : return SECOND; - case 'm' : return MINUTE; - case 'h' : return HOUR; - case 'd' : return DAY; - } - throw new AppEngineConfigException( - "Invalid age limit '" + unit + "' was specified."); + return switch (unit) { + case 's' -> SECOND; + case 'm' -> MINUTE; + case 'h' -> HOUR; + case 'd' -> DAY; + default -> throw new AppEngineConfigException( + "Invalid age limit '" + unit + "' was specified."); + }; } public char getIdent() {