diff --git a/zap/gradle/japicmp.yaml b/zap/gradle/japicmp.yaml index a8b3c09a0e8..673557a0aea 100644 --- a/zap/gradle/japicmp.yaml +++ b/zap/gradle/japicmp.yaml @@ -2,7 +2,20 @@ # # Any binary incompatible changes deemed acceptable should be added to this file. --- -packageExcludes: [] +packageExcludes: + - "ch.csnc.extension.httpclient" + - "ch.csnc.extension.ui" + - "ch.csnc.extension.util" + - "org.zaproxy.zap.extension.spider" + - "org.zaproxy.zap.spider" + - "org.zaproxy.zap.spider.filters" + - "org.zaproxy.zap.spider.parser" fieldExcludes: [] -classExcludes: [] -methodExcludes: [] \ No newline at end of file +classExcludes: + - "org.parosproxy.paros.extension.option.OptionsCertificatePanel" + - "org.parosproxy.paros.extension.option.OptionsParamCertificate" + - "org.parosproxy.paros.network.SSLConnector" +methodExcludes: + - "org.parosproxy.paros.model.OptionsParam#getCertificateParam()" + - "org.parosproxy.paros.model.OptionsParam#getExperimentalFeaturesParam()" + - "org.parosproxy.paros.model.OptionsParam#setCertificateParam(org.parosproxy.paros.extension.option.OptionsParamCertificate)" \ No newline at end of file diff --git a/zap/src/main/java/ch/csnc/extension/httpclient/AliasCertificate.java b/zap/src/main/java/ch/csnc/extension/httpclient/AliasCertificate.java deleted file mode 100644 index 1dd9d5dae70..00000000000 --- a/zap/src/main/java/ch/csnc/extension/httpclient/AliasCertificate.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * This file is part of WebScarab, an Open Web Application Security - * Project utility. For details, please see http://www.owasp.org/ - * - * Copyright (c) 2002 - 2004 Rogan Dawes - * - * Please note that this file was originally released under the - * GNU General Public License as published by the Free Software Foundation; - * either version 2 of the License, or (at your option) any later version. - * - * As of October 2014 Rogan Dawes granted the OWASP ZAP Project permission to - * redistribute this code under the Apache License, Version 2.0: - * - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package ch.csnc.extension.httpclient; - -import java.security.cert.Certificate; - -/** - * @deprecated (2.12.0) No longer in use. - */ -@Deprecated -public class AliasCertificate { - - private Certificate certificate; - private String alias; - - AliasCertificate(Certificate certificate, String alias) { - this.setCertificate(certificate); - this.setAlias(alias); - } - - public void setCertificate(Certificate certificate) { - this.certificate = certificate; - } - - public Certificate getCertificate() { - return certificate; - } - - public void setAlias(String alias) { - this.alias = alias; - } - - public String getAlias() { - return alias; - } - - public String getName() { - - String cn = getCN(); - - if (cn == null || cn.length() == 0) { - return getAlias(); - } else { - return cn + " [" + getAlias() + "]"; - } - } - - public String getCN() { - - String dn = getCertificate().toString(); - - int i = 0; - i = dn.indexOf("CN="); - if (i == -1) { - return null; - } - // get the remaining DN without CN= - dn = dn.substring(i + 3); - - char[] dncs = dn.toCharArray(); - for (i = 0; i < dncs.length; i++) { - if (dncs[i] == ',' && i > 0 && dncs[i - 1] != '\\') { - break; - } - } - return dn.substring(0, i); - } -} diff --git a/zap/src/main/java/ch/csnc/extension/httpclient/AliasKeyManager.java b/zap/src/main/java/ch/csnc/extension/httpclient/AliasKeyManager.java deleted file mode 100644 index 0370bc7076c..00000000000 --- a/zap/src/main/java/ch/csnc/extension/httpclient/AliasKeyManager.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * This file is part of WebScarab, an Open Web Application Security - * Project utility. For details, please see http://www.owasp.org/ - * - * Copyright (c) 2002 - 2004 Rogan Dawes - * - * Please note that this file was originally released under the - * GNU General Public License as published by the Free Software Foundation; - * either version 2 of the License, or (at your option) any later version. - * - * As of October 2014 Rogan Dawes granted the OWASP ZAP Project permission to - * redistribute this code under the Apache License, Version 2.0: - * - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package ch.csnc.extension.httpclient; - -import java.net.Socket; -import java.security.KeyStore; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; -import java.security.Principal; -import java.security.PrivateKey; -import java.security.UnrecoverableKeyException; -import java.security.cert.Certificate; -import java.security.cert.X509Certificate; -import javax.net.ssl.X509KeyManager; - -/** - * A KeyManager implementation that only ever selects a single alias, rather than considering the - * "best" alias for the circumstances - * - * @author rdawes - * @deprecated (2.12.0) No longer in use. - */ -@Deprecated -public class AliasKeyManager implements X509KeyManager { - - private KeyStore _ks; - private String _alias; - private String _keyPassword; - - /** - * Creates a new instance of AliasKeyManager - * - * @param ks The KeyStore that contains the keypair to use - * @param alias the alias of the certificate to use - * @param keyPassword the password for the key (not the keystore) - */ - public AliasKeyManager(KeyStore ks, String alias, String keyPassword) { - _ks = ks; - _alias = alias; - _keyPassword = keyPassword; - } - - @Override - public String chooseClientAlias(String[] str, Principal[] principal, Socket socket) { - return _alias; - } - - @Override - public String chooseServerAlias(String str, Principal[] principal, Socket socket) { - return _alias; - } - - @Override - public X509Certificate[] getCertificateChain(String alias) { - try { - Certificate[] certs = _ks.getCertificateChain(alias); - if (certs == null) return null; - X509Certificate[] x509certs = new X509Certificate[certs.length]; - for (int i = 0; i < certs.length; i++) { - x509certs[i] = (X509Certificate) certs[i]; - } - return x509certs; - } catch (KeyStoreException kse) { - kse.printStackTrace(); - return null; - } - } - - @Override - public String[] getClientAliases(String str, Principal[] principal) { - return new String[] {_alias}; - } - - @Override - public PrivateKey getPrivateKey(String alias) { - try { - return (PrivateKey) _ks.getKey(alias, _keyPassword.toCharArray()); - } catch (KeyStoreException kse) { - kse.printStackTrace(); - return null; - } catch (NoSuchAlgorithmException nsao) { - nsao.printStackTrace(); - return null; - } catch (UnrecoverableKeyException uke) { - uke.printStackTrace(); - return null; - } - } - - @Override - public String[] getServerAliases(String str, Principal[] principal) { - return new String[] {_alias}; - } -} diff --git a/zap/src/main/java/ch/csnc/extension/httpclient/PKCS11Configuration.java b/zap/src/main/java/ch/csnc/extension/httpclient/PKCS11Configuration.java deleted file mode 100644 index f0ecdd62ad9..00000000000 --- a/zap/src/main/java/ch/csnc/extension/httpclient/PKCS11Configuration.java +++ /dev/null @@ -1,214 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2014 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package ch.csnc.extension.httpclient; - -import java.io.ByteArrayInputStream; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import org.apache.commons.lang3.StringUtils; - -/** - * A representation of PKCS#11 provider configuration. Used to create configurations for instances - * of {@code sun.security.pkcs11.SunPKCS11} and {@code - * com.ibm.crypto.pkcs11impl.provider.IBMPKCS11Impl}. - * - *

Example usage: - * - *

- * - * 
- * PKCS11Configuration configuration = PKCS11Configuration.builder()
- *         .setName("Provider X")
- *         .setLibrary("/path/to/pkcs11library")
- *         .setSlotId(1)
- *         .build();
- *
- * java.security.Provider p = new sun.security.pkcs11.SunPKCS11(new ByteArrayInputStream(configuration.toString().getBytes()));
- * java.security.Security.addProvider(p);
- *
- * - *
- * - *

Note: Only the mandatory attributes, name and library, and the - * optional attributes, description, slot and slotListIndex are implemented. - * - * @see Sun - * PKCS#11 Configuration - * @see IBM - * PKCS#11 Configuration - * @deprecated (2.12.0) No longer in use. - */ -@Deprecated -public class PKCS11Configuration { - - private final String name; - - private final String library; - - private final String description; - - private final int slotId; - - private final int slotListIndex; - - private PKCS11Configuration( - String name, String library, String description, int slotId, int slotListIndex) { - super(); - - this.name = name; - this.library = library; - this.description = description; - this.slotId = slotId; - this.slotListIndex = slotListIndex; - } - - public String getName() { - return name; - } - - public String getLibrary() { - return library; - } - - public String getDescription() { - return description; - } - - public int getSlotListIndex() { - return slotListIndex; - } - - public int getSlotId() { - return slotId; - } - - @Override - public String toString() { - StringBuilder sbConfiguration = new StringBuilder(150); - sbConfiguration - .append("name = \"") - .append(escapeBackslashesAndQuotationMarks(name)) - .append("\"\n"); - sbConfiguration.append("library = ").append(library).append('\n'); - - if (description != null && !description.isEmpty()) { - sbConfiguration.append("description = ").append(description).append('\n'); - } - - if (slotListIndex != -1) { - sbConfiguration.append("slotListIndex = ").append(slotListIndex); - } else { - sbConfiguration.append("slot = ").append(slotId); - } - sbConfiguration.append('\n'); - - return sbConfiguration.toString(); - } - - private static String escapeBackslashesAndQuotationMarks(String value) { - String[] searchValues = new String[] {"\\", "\""}; - String[] replacementValues = new String[] {"\\\\", "\\\""}; - - return StringUtils.replaceEach(value, searchValues, replacementValues); - } - - public InputStream toInpuStream() { - return new ByteArrayInputStream(toString().getBytes(StandardCharsets.UTF_8)); - } - - public static PCKS11ConfigurationBuilder builder() { - return new PCKS11ConfigurationBuilder(); - } - - public static final class PCKS11ConfigurationBuilder { - - private String name; - - private String library; - - private String description; - - private int slotId; - - private int slotListIndex; - - private PCKS11ConfigurationBuilder() { - slotId = -1; - slotListIndex = 0; - } - - public PCKS11ConfigurationBuilder setName(String name) { - if (name == null || name.isEmpty()) { - throw new IllegalArgumentException("Parameter name must not be null or empty."); - } - this.name = name; - return this; - } - - public PCKS11ConfigurationBuilder setLibrary(String library) { - if (library == null || library.isEmpty()) { - throw new IllegalArgumentException("Parameter library must not be null or empty."); - } - this.library = library; - return this; - } - - public PCKS11ConfigurationBuilder setDescription(String description) { - this.description = description; - return this; - } - - public PCKS11ConfigurationBuilder setSlotListIndex(int slotListIndex) { - if (slotListIndex < 0) { - throw new IllegalArgumentException( - "Parameter slotListIndex must be greater or equal to zero."); - } - this.slotListIndex = slotListIndex; - this.slotId = -1; - return this; - } - - public final PCKS11ConfigurationBuilder setSlotId(int slotId) { - if (slotId < 0) { - throw new IllegalArgumentException( - "Parameter slotId must be greater or equal to zero."); - } - this.slotId = slotId; - this.slotListIndex = -1; - return this; - } - - public PKCS11Configuration build() { - validateBuilderState(); - return new PKCS11Configuration(name, library, description, slotId, slotListIndex); - } - - private void validateBuilderState() { - if (name == null) { - throw new IllegalStateException("A name must be set."); - } - if (library == null) { - throw new IllegalStateException("A library must be set."); - } - } - } -} diff --git a/zap/src/main/java/ch/csnc/extension/httpclient/SSLContextManager.java b/zap/src/main/java/ch/csnc/extension/httpclient/SSLContextManager.java deleted file mode 100644 index eee6b12bc77..00000000000 --- a/zap/src/main/java/ch/csnc/extension/httpclient/SSLContextManager.java +++ /dev/null @@ -1,590 +0,0 @@ -/* - * This file is part of WebScarab, an Open Web Application Security - * Project utility. For details, please see http://www.owasp.org/ - * - * Copyright (c) 2002 - 2004 Rogan Dawes - * - * Please note that this file was originally released under the - * GNU General Public License as published by the Free Software Foundation; - * either version 2 of the License, or (at your option) any later version. - * - * As of October 2014 Rogan Dawes granted the OWASP ZAP Project permission to - * redistribute this code under the Apache License, Version 2.0: - * - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package ch.csnc.extension.httpclient; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.nio.charset.StandardCharsets; -import java.security.KeyManagementException; -import java.security.KeyStore; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; -import java.security.NoSuchProviderException; -import java.security.Provider; -import java.security.SecureRandom; -import java.security.Security; -import java.security.cert.Certificate; -import java.security.cert.CertificateEncodingException; -import java.security.cert.CertificateException; -import java.security.cert.X509Certificate; -import java.util.ArrayList; -import java.util.Enumeration; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.TreeMap; -import javax.net.ssl.KeyManager; -import javax.net.ssl.SSLContext; -import javax.net.ssl.SSLSessionContext; -import javax.net.ssl.TrustManager; -import javax.net.ssl.X509TrustManager; -import org.apache.commons.codec.digest.DigestUtils; -import org.apache.commons.io.FileUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -/** - * @deprecated (2.12.0) No longer in use. - */ -@Deprecated -public class SSLContextManager { - - /** The canonical class name of Sun PKCS#11 Provider. */ - public static final String SUN_PKCS11_CANONICAL_CLASS_NAME = "sun.security.pkcs11.SunPKCS11"; - - /** The canonical class name of IBMPKCS11Impl Provider. */ - public static final String IBM_PKCS11_CANONICAL_CLASS_NAME = - "com.ibm.crypto.pkcs11impl.provider.IBMPKCS11Impl"; - - /** - * @deprecated (2.11.0) Use {@link #IBM_PKCS11_CANONICAL_CLASS_NAME} - */ - @Deprecated - public static final String IBM_PKCS11_CONONICAL_CLASS_NAME = IBM_PKCS11_CANONICAL_CLASS_NAME; - - /** The name of Sun PKCS#11 Provider. */ - private static final String SUN_PKCS11_PROVIDER_NAME = "SunPKCS11"; - - /** - * The name for providers of type PKCS#11. - * - * @see #isProviderAvailable(String) - */ - public static final String PKCS11_PROVIDER_TYPE = "PKCS11"; - - /** - * The name of the {@code KeyStore} type of Sun PKCS#11 Provider. - * - * @see KeyStore#getInstance(String, Provider) - */ - private static final String SUN_PKCS11_KEYSTORE_TYPE = "PKCS11"; - - /** - * The name of the {@code KeyStore} type of IBMPKCS11Impl Provider. - * - * @see KeyStore#getInstance(String, Provider) - */ - private static final String IBM_PKCS11_KEYSTORE_TYPE = "PKCS11IMPLKS"; - - /** - * Flag that indicates if the check for Java 9 and SunPKCS11 was already done. - * - * @see #isJava9SunPKCS11() - */ - private static Boolean java9SunPKCS11; - - private Map _contextMaps = new TreeMap<>(); - private SSLContext _noClientCertContext; - private String _defaultKey = null; - private Map> _aliasPasswords = new HashMap<>(); - private List _keyStores = new ArrayList<>(); - private Map _keyStoreDescriptions = new HashMap<>(); - private Map _keyStorePasswords = new HashMap<>(); - - private static Logger log = LogManager.getLogger(SSLContextManager.class); - - private static TrustManager[] _trustAllCerts = - new TrustManager[] { - new X509TrustManager() { - @Override - public X509Certificate[] getAcceptedIssuers() { - return null; - } - - @Override - public void checkClientTrusted(X509Certificate[] certs, String authType) {} - - @Override - public void checkServerTrusted(X509Certificate[] certs, String authType) {} - } - }; - - private int _defaultKeystoreIndex = -1; - private int _defaultAliasIndex = -1; - - /** Creates a new instance of SSLContextManager */ - public SSLContextManager() { - try { - _noClientCertContext = SSLContext.getInstance("SSL"); - _noClientCertContext.init(null, _trustAllCerts, new SecureRandom()); - } catch (NoSuchAlgorithmException nsao) { - log.error("Could not get an instance of the SSL algorithm: " + nsao.getMessage(), nsao); - } catch (KeyManagementException kme) { - log.error("Error initialising the SSL Context: " + kme.getMessage(), kme); - } - - try { - initMSCAPI(); - } catch (Exception e) { - } - } - - public boolean isProviderAvailable(String type) { - try { - if (type.equals(PKCS11_PROVIDER_TYPE)) { - try { - Class.forName(SUN_PKCS11_CANONICAL_CLASS_NAME); - return true; - } catch (Throwable ignore) { - Class.forName(IBM_PKCS11_CANONICAL_CLASS_NAME); - return true; - } - } else if (type.equals("msks")) { - Class.forName("se.assembla.jce.provider.ms.MSProvider"); - return true; - } - } catch (Throwable ignore) { - } - return false; - } - - private int addKeyStore(KeyStore ks, String description, String password) { - int index = _keyStores.indexOf(ks); - if (index == -1) { - _keyStores.add(ks); - index = _keyStores.size() - 1; - } - _keyStoreDescriptions.put(ks, description); - _keyStorePasswords.put(ks, password); - return index; - } - - public boolean removeKeyStore(int keystoreIndex) { - boolean isDefaultKeyStore = (keystoreIndex == _defaultKeystoreIndex); - KeyStore ks = _keyStores.get(keystoreIndex); - - _keyStores.remove(ks); - _keyStoreDescriptions.remove(ks); - _keyStorePasswords.remove(ks); - - if (isDefaultKeyStore) { - _defaultKeystoreIndex = -1; - _defaultAliasIndex = -1; - } - return isDefaultKeyStore; - } - - public int getKeyStoreCount() { - return _keyStores.size(); - } - - public String getKeyStoreDescription(int keystoreIndex) { - return _keyStoreDescriptions.get(_keyStores.get(keystoreIndex)); - } - - public String getKeyStorePassword(int keystoreIndex) { - return _keyStorePasswords.get(_keyStores.get(keystoreIndex)); - } - - public int getAliasCount(int keystoreIndex) { - return getAliases(_keyStores.get(keystoreIndex)).size(); - } - - public String getAliasAt(int keystoreIndex, int aliasIndex) { - return getAliases(_keyStores.get(keystoreIndex)).get(aliasIndex).getAlias(); - } - - private List getAliases(KeyStore ks) { - List aliases = new ArrayList<>(); - try { - Enumeration en = ks.aliases(); - - boolean isIbm = isIbmPKCS11Provider(); - while (en.hasMoreElements()) { - String alias = en.nextElement(); - // Sun's and IBM's KeyStore implementations behave differently... - // With IBM's KeyStore impl #getCertificate(String) returns null when - // #isKeyEntry(String) returns true. - // If IBM add all certificates and let the user choose the correct one. - if (ks.isKeyEntry(alias) || (isIbm && ks.isCertificateEntry(alias))) { - Certificate cert = ks.getCertificate(alias); - // IBM: Maybe we should check the KeyUsage? - // ((X509Certificate) cert).getKeyUsage()[0] - AliasCertificate aliasCert = new AliasCertificate(cert, alias); - aliases.add(aliasCert); - } - } - } catch (KeyStoreException kse) { - kse.printStackTrace(); - } - return aliases; - } - - public List getAliases(int ks) { - return getAliases(_keyStores.get(ks)); - } - - public Certificate getCertificate(int keystoreIndex, int aliasIndex) { - try { - KeyStore ks = _keyStores.get(keystoreIndex); - String alias = getAliasAt(keystoreIndex, aliasIndex); - return ks.getCertificate(alias); - } catch (Exception e) { - return null; - } - } - - public String getFingerPrint(Certificate cert) throws KeyStoreException { - if (!(cert instanceof X509Certificate)) { - return null; - } - - StringBuilder buff = new StringBuilder(); - X509Certificate x509 = (X509Certificate) cert; - - try { - String fingerprint = DigestUtils.md5Hex(cert.getEncoded()); - for (int i = 0; i < fingerprint.length(); i += 2) { - buff.append(fingerprint.substring(i, i + 1)).append(":"); - } - buff.deleteCharAt(buff.length() - 1); - } catch (CertificateEncodingException e) { - throw new KeyStoreException(e.getMessage()); - } - - String dn = x509.getSubjectX500Principal().getName(); - - log.info("Fingerprint is " + buff.toString().toUpperCase()); - - return buff.toString().toUpperCase() + " " + dn; - } - - public boolean isKeyUnlocked(int keystoreIndex, int aliasIndex) { - KeyStore ks = _keyStores.get(keystoreIndex); - String alias = getAliasAt(keystoreIndex, aliasIndex); - - Map pwmap = _aliasPasswords.get(ks); - if (pwmap == null) { - return false; - } - - return pwmap.containsKey(alias); - } - - public void setDefaultKey(int keystoreIndex, int aliasIndex) throws KeyStoreException { - - _defaultKeystoreIndex = keystoreIndex; - _defaultAliasIndex = aliasIndex; - - if ((_defaultKeystoreIndex == -1) || (_defaultAliasIndex == -1)) { - _defaultKey = ""; - } else { - _defaultKey = getFingerPrint(getCertificate(keystoreIndex, aliasIndex)); - } - } - - public String getDefaultKey() { - return _defaultKey; - } - - public Certificate getDefaultCertificate() { - return getCertificate(_defaultKeystoreIndex, _defaultAliasIndex); - } - - public int initMSCAPI() - throws KeyStoreException, - NoSuchProviderException, - IOException, - NoSuchAlgorithmException, - CertificateException { - try { - if (!isProviderAvailable("msks")) { - return -1; - } - - Provider mscapi = - (Provider) - Class.forName("se.assembla.jce.provider.ms.MSProvider") - .getDeclaredConstructor() - .newInstance(); - Security.addProvider(mscapi); - - // init the key store - KeyStore ks = KeyStore.getInstance("msks", "assembla"); - ks.load(null, null); - - return addKeyStore(ks, "Microsoft CAPI Store", null); - - } catch (Exception e) { - log.error("Error instantiating the MSCAPI provider: " + e.getMessage(), e); - return -1; - } - } - - /* - * public int initCryptoApi() throws KeyStoreException, - * NoSuchAlgorithmException, CertificateException, IOException{ - * - * Provider mscapi = new sun.security.mscapi.SunMSCAPI(); - * Security.addProvider(mscapi); - * - * KeyStore ks = KeyStore.getInstance("Windows-MY"); ks.load(null, null); - * - * return addKeyStore(ks, "CryptoAPI", null); } - */ - public int initPKCS11(PKCS11Configuration configuration, String kspassword) - throws IOException, - KeyStoreException, - CertificateException, - NoSuchAlgorithmException, - ClassNotFoundException, - SecurityException, - NoSuchMethodException, - IllegalArgumentException, - InstantiationException, - IllegalAccessException, - InvocationTargetException { - - if (!isProviderAvailable(PKCS11_PROVIDER_TYPE)) { - return -1; - } - - Provider pkcs11 = createPKCS11Provider(configuration); - - Security.addProvider(pkcs11); - - // init the key store - KeyStore ks = getPKCS11KeyStore(pkcs11.getName()); - ks.load(null, kspassword == null ? null : kspassword.toCharArray()); - return addKeyStore(ks, "PKCS#11: " + configuration.getName(), ""); // do not store pin code - } - - private static Provider createPKCS11Provider(PKCS11Configuration configuration) - throws ClassNotFoundException, - NoSuchMethodException, - InstantiationException, - IllegalAccessException, - InvocationTargetException, - IOException { - Provider pkcs11 = null; - if (isSunPKCS11Provider()) { - if (isJava9SunPKCS11()) { - Provider provider = Security.getProvider(SUN_PKCS11_PROVIDER_NAME); - Method configure = provider.getClass().getMethod("configure", String.class); - File configFile = File.createTempFile("pkcs11", ".cfg"); - configFile.deleteOnExit(); - FileUtils.write(configFile, configuration.toString(), StandardCharsets.UTF_8); - pkcs11 = (Provider) configure.invoke(provider, configFile.getAbsolutePath()); - } else { - pkcs11 = - createInstance( - SUN_PKCS11_CANONICAL_CLASS_NAME, - InputStream.class, - configuration.toInpuStream()); - } - } else if (isIbmPKCS11Provider()) { - pkcs11 = - createInstance( - IBM_PKCS11_CANONICAL_CLASS_NAME, - BufferedReader.class, - new BufferedReader( - new InputStreamReader(configuration.toInpuStream()))); - } - return pkcs11; - } - - private static Provider createInstance(String name, Class paramClass, Object param) - throws ClassNotFoundException, - NoSuchMethodException, - InstantiationException, - IllegalAccessException, - InvocationTargetException { - Class instanceClass = Class.forName(name); - Constructor c = instanceClass.getConstructor(new Class[] {paramClass}); - return (Provider) c.newInstance(new Object[] {param}); - } - - private static boolean isSunPKCS11Provider() { - try { - Class.forName(SUN_PKCS11_CANONICAL_CLASS_NAME); - return true; - } catch (Throwable ignore) { - } - return false; - } - - private static boolean isJava9SunPKCS11() { - if (java9SunPKCS11 != null) { - return java9SunPKCS11; - } - - java9SunPKCS11 = Boolean.FALSE; - try { - Provider provider = Security.getProvider(SUN_PKCS11_PROVIDER_NAME); - if (provider != null) { - provider.getClass().getMethod("configure", String.class); - java9SunPKCS11 = Boolean.TRUE; - } - } catch (NoSuchMethodException ignore) { - // The provider/method is available only in Java 9+. - } - return java9SunPKCS11; - } - - private static boolean isIbmPKCS11Provider() { - try { - Class.forName(IBM_PKCS11_CANONICAL_CLASS_NAME); - return true; - } catch (Throwable ignore) { - } - return false; - } - - private static KeyStore getPKCS11KeyStore(String providerName) throws KeyStoreException { - String keyStoreType = SUN_PKCS11_KEYSTORE_TYPE; - if (isIbmPKCS11Provider()) { - keyStoreType = IBM_PKCS11_KEYSTORE_TYPE; - } - return KeyStore.getInstance(keyStoreType, Security.getProvider(providerName)); - } - - public int loadPKCS12Certificate(String filename, String ksPassword) - throws IOException, KeyStoreException, CertificateException, NoSuchAlgorithmException { - - // Get Filename - File file = new File(filename); - if (!file.exists()) { - throw new FileNotFoundException(filename + " could not be found"); - } - String name = file.getName(); - - // Open the file - try (InputStream is = new FileInputStream(file)) { - // create the keystore - KeyStore ks = KeyStore.getInstance("PKCS12"); - ks.load(is, ksPassword == null ? null : ksPassword.toCharArray()); - return addKeyStore(ks, "PKCS#12: " + name, ksPassword); - } - } - - public boolean unlockKeyWithDefaultPassword(int keystoreIndex, int aliasIndex) - throws KeyManagementException, KeyStoreException { - - return unlockKey(keystoreIndex, aliasIndex, getKeyStorePassword(keystoreIndex)); - } - - public boolean unlockKey(int keystoreIndex, int aliasIndex, String keyPassword) - throws KeyStoreException, KeyManagementException { - - KeyStore ks = _keyStores.get(keystoreIndex); - String alias = getAliasAt(keystoreIndex, aliasIndex); - - AliasKeyManager akm = new AliasKeyManager(ks, alias, keyPassword); - - try { - akm.getPrivateKey(alias).toString(); - } catch (NullPointerException ex) { - log.error("Could not get private key: " + ex.getMessage(), ex); - return false; - } - - String fingerprint = getFingerPrint(getCertificate(keystoreIndex, aliasIndex)); - - if (fingerprint == null) { - log.info("No fingerprint found"); - return false; - } - - SSLContext sc; - try { - sc = SSLContext.getInstance("SSL"); - } catch (NoSuchAlgorithmException nsao) { - log.error("Could not get an instance of the SSL algorithm: " + nsao.getMessage(), nsao); - return false; - } - - sc.init(new KeyManager[] {akm}, _trustAllCerts, new SecureRandom()); - - String key = fingerprint; - if (key.indexOf(" ") > 0) { - key = key.substring(0, key.indexOf(" ")); - } - - _contextMaps.put(key, sc); - log.info("Key has been unlocked."); - - return true; - } - - public void invalidateSessions() { - invalidateSession(_noClientCertContext); - Iterator it = _contextMaps.keySet().iterator(); - while (it.hasNext()) { - invalidateSession(_contextMaps.get(it.next())); - } - } - - private void invalidateSession(SSLContext sc) { - SSLSessionContext sslsc = sc.getClientSessionContext(); - if (sslsc != null) { - int timeout = sslsc.getSessionTimeout(); - // force sessions to be timed out - sslsc.setSessionTimeout(1); - sslsc.setSessionTimeout(timeout); - } - sslsc = sc.getServerSessionContext(); - if (sslsc != null) { - int timeout = sslsc.getSessionTimeout(); - // force sessions to be timed out - sslsc.setSessionTimeout(1); - sslsc.setSessionTimeout(timeout); - } - } - - public SSLContext getSSLContext(String fingerprint) { - log.info("Requested SSLContext for " + fingerprint); - - if (fingerprint == null || fingerprint.equals("none")) { - return _noClientCertContext; - } - - if (fingerprint.indexOf(" ") > 0) { - fingerprint = fingerprint.substring(0, fingerprint.indexOf(" ")); - } - - return _contextMaps.get(fingerprint); - } -} diff --git a/zap/src/main/java/ch/csnc/extension/ui/AliasTableModel.java b/zap/src/main/java/ch/csnc/extension/ui/AliasTableModel.java deleted file mode 100644 index c262e935538..00000000000 --- a/zap/src/main/java/ch/csnc/extension/ui/AliasTableModel.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Please note that this file was originally released under the - * GNU General Public License as published by the Free Software Foundation; - * either version 2 of the License, or (at your option) any later version - * by Compass Security AG. - * - * As of October 2014 Compass Security AG granted the OWASP ZAP Project - * permission to redistribute this code under the Apache License, Version 2.0. - */ -package ch.csnc.extension.ui; - -import java.util.ArrayList; -import java.util.List; -import javax.swing.table.AbstractTableModel; - -/** - * @deprecated (2.12.0) No longer in use. - */ -@Deprecated -@SuppressWarnings("serial") -public class AliasTableModel extends AbstractTableModel { - - private static final long serialVersionUID = -4387633069248206563L; - - private int _ks = -1; - private List _aliases = new ArrayList<>(); - private ch.csnc.extension.httpclient.SSLContextManager _sslcm; - - public AliasTableModel(ch.csnc.extension.httpclient.SSLContextManager contextManager) { - _sslcm = contextManager; - } - - public void setKeystore(int ks) { - _ks = ks; - _aliases.clear(); - if (_ks > -1) { - _aliases = _sslcm.getAliases(_ks); - } - fireTableDataChanged(); - } - - public void removeKeystore() { - _ks = -1; - _aliases.clear(); - fireTableDataChanged(); - } - - public String getAlias(int row) { - return _aliases.get(row).getAlias(); - } - - @Override - public int getColumnCount() { - return 1; - } - - @Override - public int getRowCount() { - return _aliases.size(); - } - - @Override - public Object getValueAt(int row, int col) { - return _aliases.get(row).getName(); - } -} diff --git a/zap/src/main/java/ch/csnc/extension/ui/CertificateView.java b/zap/src/main/java/ch/csnc/extension/ui/CertificateView.java deleted file mode 100644 index 1952c68442e..00000000000 --- a/zap/src/main/java/ch/csnc/extension/ui/CertificateView.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Please note that this file was originally released under the - * GNU General Public License as published by the Free Software Foundation; - * either version 2 of the License, or (at your option) any later version - * by Compass Security AG. - * - * As of October 2014 Compass Security AG granted the OWASP ZAP Project - * permission to redistribute this code under the Apache License, Version 2.0. - */ -package ch.csnc.extension.ui; - -import javax.swing.GroupLayout; -import javax.swing.JButton; -import javax.swing.JScrollPane; -import javax.swing.LayoutStyle; -import org.parosproxy.paros.Constant; -import org.parosproxy.paros.view.AbstractFrame; -import org.zaproxy.zap.utils.ZapTextArea; - -/** - * @deprecated (2.12.0) No longer in use. - */ -@Deprecated -public class CertificateView extends AbstractFrame { - - private static final long serialVersionUID = -7284926693579230812L; - - /** - * Creates new form Certificate - * - * @param certificate the certificate to view/display. - */ - public CertificateView(String certificate) { - setTitle(Constant.messages.getString("view.cert.title")); - - JButton closeButton = new JButton(Constant.messages.getString("view.cert.button.close")); - closeButton.addActionListener( - e -> { - setVisible(false); - dispose(); - }); - - ZapTextArea certificateTextArea = new ZapTextArea(certificate); - certificateTextArea.setEditable(false); - - JScrollPane certificateScrollPane = new JScrollPane(certificateTextArea); - - GroupLayout layout = new GroupLayout(getContentPane()); - getContentPane().setLayout(layout); - layout.setHorizontalGroup( - layout.createParallelGroup(GroupLayout.Alignment.LEADING) - .addGroup( - GroupLayout.Alignment.TRAILING, - layout.createSequentialGroup() - .addContainerGap() - .addGroup( - layout.createParallelGroup( - GroupLayout.Alignment.TRAILING) - .addComponent( - closeButton, - GroupLayout.PREFERRED_SIZE, - 93, - GroupLayout.PREFERRED_SIZE) - .addComponent( - certificateScrollPane, - GroupLayout.DEFAULT_SIZE, - 658, - Short.MAX_VALUE)) - .addContainerGap())); - layout.setVerticalGroup( - layout.createParallelGroup(GroupLayout.Alignment.LEADING) - .addGroup( - GroupLayout.Alignment.TRAILING, - layout.createSequentialGroup() - .addContainerGap() - .addComponent( - certificateScrollPane, - GroupLayout.DEFAULT_SIZE, - 439, - Short.MAX_VALUE) - .addPreferredGap(LayoutStyle.ComponentPlacement.RELATED) - .addComponent(closeButton) - .addContainerGap())); - pack(); - - setVisible(true); - } -} diff --git a/zap/src/main/java/ch/csnc/extension/ui/DriverTableModel.java b/zap/src/main/java/ch/csnc/extension/ui/DriverTableModel.java deleted file mode 100644 index b0723e39509..00000000000 --- a/zap/src/main/java/ch/csnc/extension/ui/DriverTableModel.java +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Please note that this file was originally released under the - * GNU General Public License as published by the Free Software Foundation; - * either version 2 of the License, or (at your option) any later version - * by Compass Security AG. - * - * As of October 2014 Compass Security AG granted the OWASP ZAP Project - * permission to redistribute this code under the Apache License, Version 2.0. - */ -package ch.csnc.extension.ui; - -import java.util.Vector; -import javax.swing.table.AbstractTableModel; - -/** - * @deprecated (2.12.0) No longer in use. - */ -@Deprecated -@SuppressWarnings("serial") -public class DriverTableModel extends AbstractTableModel { - - private static final long serialVersionUID = -9114670362713975727L; - - private ch.csnc.extension.util.DriverConfiguration driverConfig; - private Vector names; - private Vector paths; - private Vector slots; - private Vector slotListIndexes; - - public DriverTableModel(ch.csnc.extension.util.DriverConfiguration driverConfig) { - this.driverConfig = driverConfig; - this.driverConfig.addChangeListener(e -> fireTableDataChanged()); - - names = driverConfig.getNames(); - paths = driverConfig.getPaths(); - slots = driverConfig.getSlots(); - slotListIndexes = driverConfig.getSlotIndexes(); - } - - @Override - public int getColumnCount() { - return 4; - } - - @Override - public int getRowCount() { - return names.size(); - } - - @Override - public Object getValueAt(int row, int column) { - if (column == 0) { - return names.get(row); - } - if (column == 1) { - return paths.get(row); - } - if (column == 2) { - return slots.get(row); - } - if (column == 3) { - return slotListIndexes.get(row); - } - - return ""; - } - - /*default*/ int getPreferredWith(int column) { - if (column == 0) { - return 75; - } - if (column == 1) { - return 300; - } - if (column == 2) { - return 15; - } - if (column == 3) { - return 15; - } - return 0; - } - - /* default */ void addDriver(String name, String path, int slot, int slotListIndex) { - names.add(name); - paths.add(path); - slots.add(slot); - slotListIndexes.add(slotListIndex); - - updateConfiguration(); - } - - /* default */ void deleteDriver(int index) { - names.remove(index); - paths.remove(index); - slots.remove(index); - slotListIndexes.remove(index); - - updateConfiguration(); - } - - private void updateConfiguration() { - driverConfig.setNames(names); - driverConfig.setPaths(paths); - driverConfig.setSlots(slots); - driverConfig.setSlotListIndexes(slotListIndexes); - driverConfig.write(); - } - - @Override - public String getColumnName(int columnNumber) { - if (columnNumber == 0) { - return "Name"; - } else if (columnNumber == 1) { - return "Path"; - } else if (columnNumber == 2) { - return "Slot"; - } else if (columnNumber == 3) { - return "SlotListIndex"; - } else { - throw new IllegalArgumentException("Invalid column number: " + columnNumber); - } - } -} diff --git a/zap/src/main/java/ch/csnc/extension/ui/DriversView.java b/zap/src/main/java/ch/csnc/extension/ui/DriversView.java deleted file mode 100644 index eb60c57016b..00000000000 --- a/zap/src/main/java/ch/csnc/extension/ui/DriversView.java +++ /dev/null @@ -1,406 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Please note that this file was originally released under the - * GNU General Public License as published by the Free Software Foundation; - * either version 2 of the License, or (at your option) any later version - * by Compass Security AG. - * - * As of October 2014 Compass Security AG granted the OWASP ZAP Project - * permission to redistribute this code under the Apache License, Version 2.0. - */ -package ch.csnc.extension.ui; - -import java.awt.Component; -import java.awt.event.ActionEvent; -import java.awt.event.ActionListener; -import javax.swing.GroupLayout; -import javax.swing.JButton; -import javax.swing.JFileChooser; -import javax.swing.JLabel; -import javax.swing.JScrollPane; -import javax.swing.JTable; -import javax.swing.LayoutStyle; -import javax.swing.SwingConstants; -import javax.swing.filechooser.FileNameExtensionFilter; -import org.parosproxy.paros.Constant; -import org.parosproxy.paros.model.Model; -import org.parosproxy.paros.view.AbstractFrame; -import org.zaproxy.zap.utils.ZapTextField; - -/** - * @deprecated (2.12.0) No longer in use. - */ -@Deprecated -public class DriversView extends AbstractFrame { - - private static final long serialVersionUID = -7502331281272992501L; - - private DriverTableModel driverTableModel; - private JTable driverTable; - - private JButton addButton; - private JButton browseButton; - private JButton closeButton; - private JButton deleteButton; - - private JScrollPane driverScrollPane; - - private JLabel fileLabel; - private ZapTextField fileTextField; - - private JLabel nameLabel; - private ZapTextField nameTextField; - - private JLabel slotLabel; - private ZapTextField slotTextField; - - private JLabel slotListIndexLabel; - private ZapTextField slotListIndexTextField; - - /** - * Creates new form Drivers - * - * @param driverConfig - */ - public DriversView(ch.csnc.extension.util.DriverConfiguration driverConfig) { - this.driverTableModel = new DriverTableModel(driverConfig); - initComponents(); - setVisible(true); - } - - /** - * This method is called from within the constructor to initialize the form. WARNING: Do NOT - * modify this code. The content of this method is always regenerated by the Form Editor. - */ - private void initComponents() { - fileLabel = new JLabel(); - fileTextField = new ZapTextField(); - browseButton = new JButton(); - nameLabel = new JLabel(); - nameTextField = new ZapTextField(); - slotLabel = new JLabel(); - slotTextField = new ZapTextField(); - slotListIndexLabel = new JLabel(); - slotListIndexTextField = new ZapTextField(); - addButton = new JButton(); - deleteButton = new JButton(); - closeButton = new JButton(); - driverScrollPane = new JScrollPane(); - driverTable = new JTable(); - - setTitle(Constant.messages.getString("certificates.pkcs11.drivers.title")); - fileLabel.setText(Constant.messages.getString("certificates.pkcs11.drivers.label.path")); - - browseButton.setText( - Constant.messages.getString("certificates.pkcs11.drivers.button.browse")); - browseButton.addActionListener( - new ActionListener() { - @Override - public void actionPerformed(ActionEvent evt) { - browseButtonActionPerformed(evt); - } - }); - - nameLabel.setText(Constant.messages.getString("certificates.pkcs11.drivers.label.name")); - - slotLabel.setText(Constant.messages.getString("certificates.pkcs11.drivers.label.slot")); - - slotListIndexLabel.setText( - Constant.messages.getString("certificates.pkcs11.drivers.label.slotIndex")); - - addButton.setText(Constant.messages.getString("certificates.pkcs11.drivers.button.add")); - addButton.addActionListener( - new ActionListener() { - @Override - public void actionPerformed(ActionEvent evt) { - addButtonActionPerformed(evt); - } - }); - - deleteButton.setText( - Constant.messages.getString("certificates.pkcs11.drivers.button.delete")); - deleteButton.addActionListener( - new ActionListener() { - @Override - public void actionPerformed(ActionEvent evt) { - deleteButtonActionPerformed(evt); - } - }); - - closeButton.setText( - Constant.messages.getString("certificates.pkcs11.drivers.button.close")); - closeButton.addActionListener( - new ActionListener() { - @Override - public void actionPerformed(ActionEvent evt) { - closeButtonActionPerformed(evt); - } - }); - - driverTable.setModel(driverTableModel); - driverScrollPane.setViewportView(driverTable); - - // When experimental SlotListIndex support is used, the slotTextField is disabled (and vice - // versa), - // as only one of these parameters is actually used. - if (!Model.getSingleton() - .getOptionsParam() - .getExperimentalFeaturesParam() - .isExperimentalSliSupportEnabled()) { - slotTextField.setEnabled(false); - } - - final GroupLayout layout = new GroupLayout(getContentPane()); - getContentPane().setLayout(layout); - layout.setHorizontalGroup( - layout.createParallelGroup(GroupLayout.Alignment.LEADING) - .addGroup( - layout.createSequentialGroup() - .addContainerGap() - .addGroup( - layout.createParallelGroup( - GroupLayout.Alignment.LEADING) - .addComponent(fileLabel) - .addComponent(nameLabel) - .addComponent(slotLabel) - .addComponent(slotListIndexLabel) - .addGroup( - layout.createSequentialGroup() - .addGroup( - layout.createParallelGroup( - GroupLayout - .Alignment - .TRAILING, - false) - .addComponent( - nameTextField, - GroupLayout - .Alignment - .LEADING) - .addComponent( - slotTextField, - GroupLayout - .Alignment - .LEADING) - .addComponent( - slotListIndexTextField, - GroupLayout - .Alignment - .LEADING) - .addComponent( - fileTextField, - GroupLayout - .Alignment - .LEADING, - GroupLayout - .DEFAULT_SIZE, - 322, - Short - .MAX_VALUE)) - .addPreferredGap( - LayoutStyle - .ComponentPlacement - .RELATED) - .addGroup( - layout.createParallelGroup( - GroupLayout - .Alignment - .LEADING) - .addComponent( - addButton, - GroupLayout - .DEFAULT_SIZE, - 80, - Short - .MAX_VALUE) - .addComponent( - browseButton)))) - .addContainerGap(165, Short.MAX_VALUE)) - .addGroup( - GroupLayout.Alignment.TRAILING, - layout.createSequentialGroup() - .addGap(499, 499, 499) - .addComponent( - closeButton, - GroupLayout.DEFAULT_SIZE, - 74, - Short.MAX_VALUE) - .addContainerGap()) - .addGroup( - layout.createSequentialGroup() - .addContainerGap() - .addComponent( - driverScrollPane, - GroupLayout.DEFAULT_SIZE, - 561, - Short.MAX_VALUE) - .addContainerGap()) - .addGroup( - GroupLayout.Alignment.TRAILING, - layout.createSequentialGroup() - .addContainerGap(499, Short.MAX_VALUE) - .addComponent(deleteButton) - .addContainerGap())); - layout.setVerticalGroup( - layout.createParallelGroup(GroupLayout.Alignment.LEADING) - .addGroup( - GroupLayout.Alignment.TRAILING, - layout.createSequentialGroup() - .addContainerGap() - .addComponent(fileLabel) - .addPreferredGap(LayoutStyle.ComponentPlacement.RELATED) - .addGroup( - layout.createParallelGroup( - GroupLayout.Alignment.LEADING, - false) - .addComponent( - browseButton, 0, 0, Short.MAX_VALUE) - .addComponent(fileTextField)) - .addPreferredGap(LayoutStyle.ComponentPlacement.RELATED) - .addComponent(nameLabel) - .addPreferredGap(LayoutStyle.ComponentPlacement.RELATED) - .addGroup( - layout.createParallelGroup( - GroupLayout.Alignment.BASELINE) - .addComponent( - nameTextField, - GroupLayout.PREFERRED_SIZE, - GroupLayout.DEFAULT_SIZE, - GroupLayout.PREFERRED_SIZE)) - .addPreferredGap(LayoutStyle.ComponentPlacement.RELATED) - .addComponent(slotLabel) - .addPreferredGap(LayoutStyle.ComponentPlacement.RELATED) - .addGroup( - layout.createParallelGroup( - GroupLayout.Alignment.BASELINE) - .addComponent( - slotTextField, - GroupLayout.PREFERRED_SIZE, - GroupLayout.DEFAULT_SIZE, - GroupLayout.PREFERRED_SIZE)) - .addGap(28, 28, 28) - .addComponent(slotListIndexLabel) - .addPreferredGap(LayoutStyle.ComponentPlacement.RELATED) - .addGroup( - layout.createParallelGroup( - GroupLayout.Alignment.BASELINE) - .addComponent( - slotListIndexTextField, - GroupLayout.PREFERRED_SIZE, - GroupLayout.DEFAULT_SIZE, - GroupLayout.PREFERRED_SIZE) - .addComponent( - addButton, - GroupLayout.PREFERRED_SIZE, - 19, - GroupLayout.PREFERRED_SIZE)) - .addGap(28, 28, 28) - .addComponent( - driverScrollPane, - GroupLayout.PREFERRED_SIZE, - 195, - GroupLayout.PREFERRED_SIZE) - .addPreferredGap(LayoutStyle.ComponentPlacement.RELATED) - .addComponent(deleteButton) - .addPreferredGap( - LayoutStyle.ComponentPlacement.RELATED, - 9, - Short.MAX_VALUE) - .addComponent( - closeButton, - GroupLayout.PREFERRED_SIZE, - 10, - GroupLayout.PREFERRED_SIZE) - .addContainerGap())); - - layout.linkSize( - SwingConstants.VERTICAL, - new Component[] { - addButton, browseButton, closeButton, deleteButton, fileTextField, nameTextField - }); - - for (int i = 0; i < driverTableModel.getColumnCount(); i++) { - driverTable - .getColumnModel() - .getColumn(i) - .setPreferredWidth(driverTableModel.getPreferredWith(i)); - } - - pack(); - } - - private void browseButtonActionPerformed(ActionEvent evt) { - final JFileChooser fc = new JFileChooser(); - // TODO Support so and dynlib files as well - fc.setFileFilter(new FileNameExtensionFilter("DLL/dylib", "dll", "dylib")); - - final int state = fc.showOpenDialog(null); - - if (state == JFileChooser.APPROVE_OPTION) { - fileTextField.setText(fc.getSelectedFile().toString()); - } - } - - private void addButtonActionPerformed(ActionEvent evt) { - final String name = nameTextField.getText(); - final String file = fileTextField.getText(); - int slot = -1; - int slotListindex = -1; - try { - slot = Integer.parseInt(slotTextField.getText()); - } catch (final Exception e) { - slotTextField.setText("0"); - } - try { - slotListindex = Integer.parseInt(slotListIndexTextField.getText()); - } catch (final Exception e) { - slotListIndexTextField.setText("0"); - } - - if (name != null - && name.trim().length() > 0 - && file != null - && file.trim().length() > 0 - && slot > -1 - && slotListindex > -1) { - driverTableModel.addDriver(name, file, slot, slotListindex); - - nameTextField.setText(""); - fileTextField.setText(""); - slotTextField.setText("0"); - slotListIndexTextField.setText("0"); - } - } - - private void deleteButtonActionPerformed(ActionEvent evt) { - final int selrow = driverTable.getSelectedRow(); - if (selrow > -1) { - driverTableModel.deleteDriver(selrow); - - nameTextField.setText(""); - fileTextField.setText(""); - slotTextField.setText("0"); - slotListIndexTextField.setText("0"); - } - } - - private void closeButtonActionPerformed(ActionEvent evt) { - setVisible(false); - dispose(); - } -} diff --git a/zap/src/main/java/ch/csnc/extension/util/DriverConfiguration.java b/zap/src/main/java/ch/csnc/extension/util/DriverConfiguration.java deleted file mode 100644 index 3cbee2502e6..00000000000 --- a/zap/src/main/java/ch/csnc/extension/util/DriverConfiguration.java +++ /dev/null @@ -1,187 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Please note that this file was originally released under the - * GNU General Public License as published by the Free Software Foundation; - * either version 2 of the License, or (at your option) any later version - * by Compass Security AG. - * - * As of October 2014 Compass Security AG granted the OWASP ZAP Project - * permission to redistribute this code under the Apache License, Version 2.0. - */ -package ch.csnc.extension.util; - -import java.io.File; -import java.net.URL; -import java.util.List; -import java.util.Vector; -import javax.swing.event.ChangeEvent; -import javax.swing.event.ChangeListener; -import javax.swing.event.EventListenerList; -import org.apache.commons.configuration.ConfigurationException; -import org.apache.commons.configuration.HierarchicalConfiguration; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.zaproxy.zap.utils.ZapXmlConfiguration; - -/** - * @deprecated (2.12.0) No longer in use. - */ -@Deprecated -public class DriverConfiguration { - private File file = null; - private URL url; - - private Vector names; - private Vector paths; - private Vector slots; - private Vector slotListIndexes; - - private final Logger logger = LogManager.getLogger(this.getClass()); - - private EventListenerList eventListeners = new EventListenerList(); - private ChangeEvent changeEvent; - - public DriverConfiguration(URL url) { - this.url = url; - load(); - } - - public DriverConfiguration(File file) { - this.file = file; - load(); - } - - private void load() { - names = new Vector<>(); - paths = new Vector<>(); - slots = new Vector<>(); - slotListIndexes = new Vector<>(); - - try { - ZapXmlConfiguration configuration = - file != null ? new ZapXmlConfiguration(file) : new ZapXmlConfiguration(url); - List drivers = configuration.configurationsAt("driver"); - for (HierarchicalConfiguration driver : drivers) { - names.add(driver.getString("name", "")); - paths.add(driver.getString("path", "")); - slots.add(getInt(driver.getString("slot"))); - slotListIndexes.add(getInt(driver.getString("slotListIndex"))); - } - - } catch (ConfigurationException e) { - logger.error("Failed to read the configuration from " + (file != null ? file : url), e); - } - } - - /** - * Gets an integer from the given string. - * - *

If the given string is {@code null} or does not have an integer, zero is returned. - * - * @param string the string with the integer value - * @return an integer - */ - private int getInt(String string) { - if (string != null) { - try { - return Integer.parseInt(string); - } catch (NumberFormatException e) { - logger.error("Failed to extract an integer from: " + string); - } - } - return 0; - } - - public void write() { - if (file == null) { - fireStateChanged(); - return; - } - - ZapXmlConfiguration configuration = new ZapXmlConfiguration(); - configuration.setRootElementName("driverConfiguration"); - - for (int i = 0; i < names.size(); i++) { - String baseKey = "driver(" + i + ")."; - configuration.setProperty(baseKey + "name", names.get(i)); - configuration.setProperty(baseKey + "path", paths.get(i)); - configuration.setProperty(baseKey + "slot", slots.get(i).toString()); - configuration.setProperty(baseKey + "slotListIndex", slotListIndexes.get(i).toString()); - } - - try { - configuration.save(file); - } catch (ConfigurationException e) { - logger.error("Failed to save driver configuration to " + file, e); - } - - fireStateChanged(); - } - - private void fireStateChanged() { - Object[] listeners = eventListeners.getListenerList(); - for (int i = listeners.length - 2; i >= 0; i -= 2) { - if (listeners[i] == ChangeListener.class) { - if (changeEvent == null) { - changeEvent = new ChangeEvent(this); - } - ((ChangeListener) listeners[i + 1]).stateChanged(changeEvent); - } - } - } - - public Vector getNames() { - return names; - } - - public void setNames(Vector names) { - this.names = names; - } - - public Vector getPaths() { - return paths; - } - - public void setPaths(Vector paths) { - this.paths = paths; - } - - public Vector getSlots() { - return slots; - } - - public void setSlots(Vector slots) { - this.slots = slots; - } - - public Vector getSlotIndexes() { - return slotListIndexes; - } - - public void setSlotListIndexes(Vector slotListIndexes) { - this.slotListIndexes = slotListIndexes; - } - - public void addChangeListener(ChangeListener listener) { - eventListeners.add(ChangeListener.class, listener); - } - - public void removeChangeListener(ChangeListener listener) { - eventListeners.remove(ChangeListener.class, listener); - } -} diff --git a/zap/src/main/java/ch/csnc/extension/util/OptionsParamExperimentalSliSupport.java b/zap/src/main/java/ch/csnc/extension/util/OptionsParamExperimentalSliSupport.java deleted file mode 100644 index a5221e6a4d0..00000000000 --- a/zap/src/main/java/ch/csnc/extension/util/OptionsParamExperimentalSliSupport.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2011 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package ch.csnc.extension.util; - -import org.parosproxy.paros.common.AbstractParam; - -/** - * @deprecated (2.12.0) No longer in use. - */ -@Deprecated -public class OptionsParamExperimentalSliSupport extends AbstractParam { - - public static final String EXPERIMENTAL_SLOT_LIST_INDEXES = - "certificate.experimentalSlotListIndex"; - private boolean expSliSupportEnabled = false; - - public OptionsParamExperimentalSliSupport() {} - - @Override - protected void parse() { - expSliSupportEnabled = getBoolean(EXPERIMENTAL_SLOT_LIST_INDEXES, false); - } - - /** - * @deprecated (2.11.0) Use {@link #isExperimentalSliSupportEnabled()} - */ - @Deprecated - public boolean isExerimentalSliSupportEnabled() { - return isExperimentalSliSupportEnabled(); - } - - public boolean isExperimentalSliSupportEnabled() { - return expSliSupportEnabled; - } - - public void setSlotListIndexSupport(boolean expSliSupportEnabled) { - this.expSliSupportEnabled = expSliSupportEnabled; - getConfig().setProperty(EXPERIMENTAL_SLOT_LIST_INDEXES, expSliSupportEnabled); - } -} diff --git a/zap/src/main/java/org/parosproxy/paros/extension/option/OptionsCertificatePanel.java b/zap/src/main/java/org/parosproxy/paros/extension/option/OptionsCertificatePanel.java deleted file mode 100644 index 528ec46be9e..00000000000 --- a/zap/src/main/java/org/parosproxy/paros/extension/option/OptionsCertificatePanel.java +++ /dev/null @@ -1,1374 +0,0 @@ -/* - * - * Paros and its related class files. - * - * Paros is an HTTP/HTTPS proxy for assessing web application security. - * Copyright (C) 2003-2004 Chinotec Technologies Company - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the Clarified Artistic License - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Clarified Artistic License for more details. - * - * You should have received a copy of the Clarified Artistic License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - */ -// ZAP: 2011/04/16 i18n -// ZAP: 2012/04/25 Added @Override annotation to the appropriate methods. -// ZAP: 2013/01/23 Clean up of exception handling/logging. -// ZAP: 2013/03/03 Issue 546: Remove all template Javadoc comments -// ZAP: 2013/12/03 Issue 933: Automatically determine install dir -// ZAP: 2014/03/23 Issue 412: Enable unsafe SSL/TLS renegotiation option not saved -// ZAP: 2014/08/14 Issue 1184: Improve support for IBM JDK -// ZAP: 2016/06/28: File chooser for PKCS#12 files now also accepts .pfx files -// ZAP: 2017/01/09 Remove method no longer needed. -// ZAP: 2017/01/23 Select first alias of selected keystore -// ZAP: 2017/08/16 Tidy up usage of CertificateView. -// ZAP: 2017/08/16 Show error message if failed to activate the certificate. -// ZAP: 2017/08/17 Reduce code duplication when showing cert/keystore errors -// ZAP: 2017/12/12 Use first alias by default (Issue 3879). -// ZAP: 2017/12/13 Do not allow to edit the name/key of active cert. -// ZAP: 2018/02/14 Remove unnecessary boxing / unboxing -// ZAP: 2018/03/29 Use FileNameExtensionFilter. -// ZAP: 2018/07/12 Fallback to bundled drivers.xml file. -// ZAP: 2018/09/19 GUI support for setting client certificate from CLI -// ZAP: 2019/06/01 Normalise line endings. -// ZAP: 2019/06/05 Normalise format/style. -// ZAP: 2020/11/26 Use Log4j 2 classes for logging. -// ZAP: 2022/05/21 Remove unsafe SSL/TLS renegotiation option. -// ZAP: 2022/05/29 Deprecate the class. -// ZAP: 2022/08/05 Address warns with Java 18 (Issue 7389). -// ZAP: 2022/09/21 Use format specifiers instead of concatenation when logging. -package org.parosproxy.paros.extension.option; - -// TODO: Buttons should be gray -import java.awt.CardLayout; -import java.lang.reflect.InvocationTargetException; -import java.net.URI; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.security.KeyStoreException; -import java.security.ProviderException; -import java.security.cert.Certificate; -import javax.swing.DefaultListModel; -import javax.swing.JDialog; -import javax.swing.JFileChooser; -import javax.swing.JFrame; -import javax.swing.JOptionPane; -import javax.swing.JPanel; -import javax.swing.JPasswordField; -import javax.swing.event.ListSelectionEvent; -import javax.swing.event.ListSelectionListener; -import javax.swing.filechooser.FileNameExtensionFilter; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.jdesktop.swingx.JXHyperlink; -import org.parosproxy.paros.Constant; -import org.parosproxy.paros.model.Model; -import org.parosproxy.paros.model.OptionsParam; -import org.parosproxy.paros.view.AbstractParamPanel; -import org.zaproxy.zap.utils.ZapTextField; - -/** - * @deprecated (2.12.0) No longer in use. - */ -@Deprecated -@SuppressWarnings("serial") -public class OptionsCertificatePanel extends AbstractParamPanel { - - private static final long serialVersionUID = 4350957038174673492L; - - // Maximum number of login attempts per smartcard - private static final int MAX_LOGIN_ATTEMPTS = 3; - - private javax.swing.JButton addPkcs11Button; - private javax.swing.JButton addPkcs12Button; - private javax.swing.JScrollPane aliasScrollPane; - private javax.swing.JTable aliasTable; - private javax.swing.JButton browseButton; - private javax.swing.JLabel certificateLabel; - private javax.swing.JPanel certificatePanel; - private ZapTextField certificateTextField; - private javax.swing.JTabbedPane certificatejTabbedPane; - private javax.swing.JButton deleteButton; - private javax.swing.JButton driverButton; - private javax.swing.JComboBox driverComboBox; - private javax.swing.JLabel driverLabel; - private javax.swing.JLabel fileLabel; - private ZapTextField fileTextField; - private javax.swing.JList keyStoreList; - private javax.swing.JPanel keyStorePanel; - private javax.swing.JScrollPane keyStoreScrollPane; - private javax.swing.JLabel passwordPkcs11Label; - private javax.swing.JLabel passwordPkcs12Label; - private javax.swing.JPanel pkcs11Panel; - private javax.swing.JPasswordField pkcs11PasswordField; - private javax.swing.JPanel pkcs12Panel; - private javax.swing.JPasswordField pkcs12PasswordField; - private javax.swing.JButton setActiveButton; - private javax.swing.JButton showActiveCertificateButton; - private javax.swing.JButton showAliasButton; - private javax.swing.JLabel textLabel; - private javax.swing.JCheckBox useClientCertificateCheckBox; - private javax.swing.JCheckBox usePkcs11ExperimentalSliSupportCheckBox; - - private ch.csnc.extension.httpclient.SSLContextManager contextManager; - private DefaultListModel keyStoreListModel; - private ch.csnc.extension.ui.AliasTableModel aliasTableModel; - private ch.csnc.extension.util.DriverConfiguration driverConfig; - - // Issue 182 - private boolean retry = true; - // Used if certificate is set from commandline - private boolean overrideEnableClientCertificate = false; - - // Keep track of login attempts on PKCS11 smartcards to avoid blocking the smartcard - private static int login_attempts = 0; - - private static final Logger logger = LogManager.getLogger(OptionsCertificatePanel.class); - - public OptionsCertificatePanel() { - super(); - initialize(); - } - - /** This method initializes this */ - private void initialize() { - - contextManager = - Model.getSingleton().getOptionsParam().getCertificateParam().getSSLContextManager(); - - keyStoreListModel = new DefaultListModel<>(); - aliasTableModel = new ch.csnc.extension.ui.AliasTableModel(contextManager); - - this.setLayout(new CardLayout()); - this.setName(Constant.messages.getString("options.cert.title")); - - JPanel certificatePanel = getPanelCertificate(); - this.add(certificatePanel, certificatePanel.getName()); - - driverConfig = createDriverConfiguration(); - updateDriverComboBox(); - driverConfig.addChangeListener(e -> updateDriverComboBox()); - - Certificate cert = contextManager.getDefaultCertificate(); - if (cert != null) { - certificateTextField.setText(cert.toString()); - } - - if (contextManager.getKeyStoreCount() != 0) { - overrideEnableClientCertificate = true; - } - } - - private static ch.csnc.extension.util.DriverConfiguration createDriverConfiguration() { - String fileName = "drivers.xml"; - Path path = Paths.get(Constant.getZapInstall(), "xml", fileName); - if (Files.exists(path)) { - return new ch.csnc.extension.util.DriverConfiguration(path.toFile()); - } - return new ch.csnc.extension.util.DriverConfiguration( - OptionsCertificatePanel.class.getResource( - "/org/zaproxy/zap/resources/" + fileName)); - } - - private void updateDriverComboBox() { - driverComboBox.removeAllItems(); - for (String name : driverConfig.getNames()) { - driverComboBox.addItem(name); - } - driverComboBox.repaint(); - } - - /** - * This method initializes panelCertificate - * - * @return javax.swing.JPanel - */ - private JPanel getPanelCertificate() { - if (certificatePanel == null) { - - // ************************************************************************** - // begin netbeans code - // ************************************************************************** - certificatePanel = new javax.swing.JPanel(); - certificatejTabbedPane = new javax.swing.JTabbedPane(); - keyStorePanel = new javax.swing.JPanel(); - setActiveButton = new javax.swing.JButton(); - showAliasButton = new javax.swing.JButton(); - aliasScrollPane = new javax.swing.JScrollPane(); - aliasTable = new javax.swing.JTable(); - deleteButton = new javax.swing.JButton(); - keyStoreScrollPane = new javax.swing.JScrollPane(); - keyStoreList = new javax.swing.JList<>(); - pkcs12Panel = new javax.swing.JPanel(); - fileLabel = new javax.swing.JLabel(); - fileTextField = new ZapTextField(); - browseButton = new javax.swing.JButton(); - passwordPkcs12Label = new javax.swing.JLabel(); - addPkcs12Button = new javax.swing.JButton(); - pkcs12PasswordField = new javax.swing.JPasswordField(); - pkcs11Panel = new javax.swing.JPanel(); - driverLabel = new javax.swing.JLabel(); - driverComboBox = new javax.swing.JComboBox<>(); - driverButton = new javax.swing.JButton(); - passwordPkcs11Label = new javax.swing.JLabel(); - addPkcs11Button = new javax.swing.JButton(); - pkcs11PasswordField = new javax.swing.JPasswordField(); - useClientCertificateCheckBox = new javax.swing.JCheckBox(); - textLabel = new javax.swing.JLabel(); - certificateLabel = new javax.swing.JLabel(); - certificateTextField = new ZapTextField(); - showActiveCertificateButton = new javax.swing.JButton(); - usePkcs11ExperimentalSliSupportCheckBox = new javax.swing.JCheckBox(); - - certificatejTabbedPane.setEnabled(false); - - setActiveButton.setText(Constant.messages.getString("options.cert.button.setactive")); - setActiveButton.setEnabled(false); - setActiveButton.addActionListener( - new java.awt.event.ActionListener() { - @Override - public void actionPerformed(java.awt.event.ActionEvent evt) { - try { - setActiveButtonActionPerformed(evt); - } catch (ProviderException e) { - showKeyStoreCertError(e.toString()); - logger.error(e.getMessage(), e); - } - } - }); - - showAliasButton.setText("->"); - showAliasButton.setEnabled(false); - showAliasButton.setMargin(new java.awt.Insets(2, 2, 2, 2)); - showAliasButton.addActionListener( - new java.awt.event.ActionListener() { - @Override - public void actionPerformed(java.awt.event.ActionEvent evt) { - showAliasButtonActionPerformed(evt); - } - }); - - aliasTable.setModel(aliasTableModel); - aliasTable.setTableHeader(null); - aliasScrollPane.setViewportView(aliasTable); - - deleteButton.setText(Constant.messages.getString("options.cert.button.delete")); - deleteButton.setEnabled(false); - deleteButton.addActionListener( - new java.awt.event.ActionListener() { - @Override - public void actionPerformed(java.awt.event.ActionEvent evt) { - deleteButtonActionPerformed(evt); - } - }); - - keyStoreList.setModel(keyStoreListModel); - keyStoreList.addListSelectionListener( - new ListSelectionListener() { - - @Override - public void valueChanged(ListSelectionEvent evt) { - keyStoreListSelectionChanged(); - } - }); - keyStoreScrollPane.setViewportView(keyStoreList); - - javax.swing.GroupLayout keyStorePanelLayout = - new javax.swing.GroupLayout(keyStorePanel); - keyStorePanel.setLayout(keyStorePanelLayout); - keyStorePanelLayout.setHorizontalGroup( - keyStorePanelLayout - .createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addGroup( - javax.swing.GroupLayout.Alignment.TRAILING, - keyStorePanelLayout - .createSequentialGroup() - .addGroup( - keyStorePanelLayout - .createParallelGroup( - javax.swing.GroupLayout - .Alignment.LEADING) - .addComponent(deleteButton) - .addComponent( - keyStoreScrollPane, - javax.swing.GroupLayout - .DEFAULT_SIZE, - 181, - Short.MAX_VALUE)) - .addPreferredGap( - javax.swing.LayoutStyle.ComponentPlacement - .RELATED) - .addGroup( - keyStorePanelLayout - .createParallelGroup( - javax.swing.GroupLayout - .Alignment.LEADING) - .addGroup( - keyStorePanelLayout - .createSequentialGroup() - .addComponent( - setActiveButton) - .addPreferredGap( - javax.swing - .LayoutStyle - .ComponentPlacement - .RELATED, - 100, - Short.MAX_VALUE) - .addComponent( - showAliasButton)) - .addComponent( - aliasScrollPane, - javax.swing.GroupLayout - .DEFAULT_SIZE, - 202, - Short.MAX_VALUE)))); - keyStorePanelLayout.setVerticalGroup( - keyStorePanelLayout - .createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addGroup( - javax.swing.GroupLayout.Alignment.TRAILING, - keyStorePanelLayout - .createSequentialGroup() - .addGroup( - keyStorePanelLayout - .createParallelGroup( - javax.swing.GroupLayout - .Alignment.LEADING) - .addComponent( - aliasScrollPane, - 0, - 0, - Short.MAX_VALUE) - .addComponent( - keyStoreScrollPane, - javax.swing.GroupLayout - .DEFAULT_SIZE, - 95, - Short.MAX_VALUE)) - .addPreferredGap( - javax.swing.LayoutStyle.ComponentPlacement - .RELATED) - .addGroup( - keyStorePanelLayout - .createParallelGroup( - javax.swing.GroupLayout - .Alignment.BASELINE) - .addComponent(deleteButton) - .addComponent( - setActiveButton, - javax.swing.GroupLayout - .PREFERRED_SIZE, - 18, - javax.swing.GroupLayout - .PREFERRED_SIZE) - .addComponent(showAliasButton)))); - - keyStorePanelLayout.linkSize( - javax.swing.SwingConstants.VERTICAL, - new java.awt.Component[] {deleteButton, setActiveButton, showAliasButton}); - - certificatejTabbedPane.addTab( - Constant.messages.getString("options.cert.tab.keystore"), keyStorePanel); - - fileLabel.setText(Constant.messages.getString("options.cert.label.file")); - - browseButton.setText(Constant.messages.getString("options.cert.button.browse")); - browseButton.addActionListener( - new java.awt.event.ActionListener() { - @Override - public void actionPerformed(java.awt.event.ActionEvent evt) { - browseButtonActionPerformed(evt); - } - }); - - passwordPkcs12Label.setText(Constant.messages.getString("options.cert.label.password")); - - addPkcs12Button.setText(Constant.messages.getString("options.cert.button.keystore")); - addPkcs12Button.addActionListener( - new java.awt.event.ActionListener() { - @Override - public void actionPerformed(java.awt.event.ActionEvent evt) { - addPkcs12ButtonActionPerformed(evt); - } - }); - - javax.swing.GroupLayout pkcs12PanelLayout = new javax.swing.GroupLayout(pkcs12Panel); - pkcs12Panel.setLayout(pkcs12PanelLayout); - pkcs12PanelLayout.setHorizontalGroup( - pkcs12PanelLayout - .createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addGroup( - pkcs12PanelLayout - .createSequentialGroup() - .addGroup( - pkcs12PanelLayout - .createParallelGroup( - javax.swing.GroupLayout - .Alignment.LEADING) - .addGroup( - javax.swing.GroupLayout - .Alignment.TRAILING, - pkcs12PanelLayout - .createSequentialGroup() - .addContainerGap() - .addComponent( - fileTextField, - javax.swing - .GroupLayout - .DEFAULT_SIZE, - 296, - Short.MAX_VALUE) - .addPreferredGap( - javax.swing - .LayoutStyle - .ComponentPlacement - .RELATED) - .addComponent( - browseButton)) - .addGroup( - pkcs12PanelLayout - .createSequentialGroup() - .addGap(12, 12, 12) - .addComponent( - fileLabel)) - .addGroup( - pkcs12PanelLayout - .createSequentialGroup() - .addContainerGap() - .addComponent( - passwordPkcs12Label)) - .addGroup( - javax.swing.GroupLayout - .Alignment.TRAILING, - pkcs12PanelLayout - .createSequentialGroup() - .addContainerGap( - 270, - Short.MAX_VALUE) - .addComponent( - addPkcs12Button)) - .addGroup( - pkcs12PanelLayout - .createSequentialGroup() - .addContainerGap() - .addComponent( - pkcs12PasswordField, - javax.swing - .GroupLayout - .DEFAULT_SIZE, - 369, - Short - .MAX_VALUE))) - .addContainerGap())); - pkcs12PanelLayout.setVerticalGroup( - pkcs12PanelLayout - .createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addGroup( - javax.swing.GroupLayout.Alignment.TRAILING, - pkcs12PanelLayout - .createSequentialGroup() - .addComponent(fileLabel) - .addPreferredGap( - javax.swing.LayoutStyle.ComponentPlacement - .RELATED) - .addGroup( - pkcs12PanelLayout - .createParallelGroup( - javax.swing.GroupLayout - .Alignment.BASELINE) - .addComponent(browseButton) - .addComponent( - fileTextField, - javax.swing.GroupLayout - .PREFERRED_SIZE, - javax.swing.GroupLayout - .DEFAULT_SIZE, - javax.swing.GroupLayout - .PREFERRED_SIZE)) - .addPreferredGap( - javax.swing.LayoutStyle.ComponentPlacement - .RELATED) - .addComponent(passwordPkcs12Label) - .addPreferredGap( - javax.swing.LayoutStyle.ComponentPlacement - .RELATED) - .addComponent( - pkcs12PasswordField, - javax.swing.GroupLayout.PREFERRED_SIZE, - javax.swing.GroupLayout.DEFAULT_SIZE, - javax.swing.GroupLayout.PREFERRED_SIZE) - .addPreferredGap( - javax.swing.LayoutStyle.ComponentPlacement - .RELATED) - .addComponent(addPkcs12Button) - .addGap(70, 70, 70))); - - pkcs12PanelLayout.linkSize( - javax.swing.SwingConstants.VERTICAL, - new java.awt.Component[] { - addPkcs12Button, browseButton, fileTextField, pkcs12PasswordField - }); - - certificatejTabbedPane.addTab( - Constant.messages.getString("options.cert.tab.pkcs"), pkcs12Panel); - - driverLabel.setText(Constant.messages.getString("options.cert.label.driver")); - - driverButton.setText("..."); - driverButton.setMargin(new java.awt.Insets(2, 5, 2, 5)); - driverButton.addActionListener( - new java.awt.event.ActionListener() { - @Override - public void actionPerformed(java.awt.event.ActionEvent evt) { - driverButtonActionPerformed(evt); - } - }); - - passwordPkcs11Label.setText(Constant.messages.getString("options.cert.label.pincode")); - - addPkcs11Button.setText(Constant.messages.getString("options.cert.button.pkcs11")); - addPkcs11Button.addActionListener( - new java.awt.event.ActionListener() { - @Override - public void actionPerformed(java.awt.event.ActionEvent evt) { - addPkcs11ButtonActionPerformed(evt); - } - }); - - usePkcs11ExperimentalSliSupportCheckBox.setText( - Constant.messages.getString( - "certificates.pkcs11.label.experimentalSliSupport")); - usePkcs11ExperimentalSliSupportCheckBox.setBorder( - javax.swing.BorderFactory.createEmptyBorder(0, 0, 0, 0)); - usePkcs11ExperimentalSliSupportCheckBox.setMargin(new java.awt.Insets(0, 0, 0, 0)); - usePkcs11ExperimentalSliSupportCheckBox.addActionListener( - new java.awt.event.ActionListener() { - @Override - public void actionPerformed(java.awt.event.ActionEvent evt) { - usePkcs11ExperimentalSliSupportCheckBoxActionPerformed(evt); - } - }); - - javax.swing.GroupLayout pkcs11PanelLayout = new javax.swing.GroupLayout(pkcs11Panel); - pkcs11Panel.setLayout(pkcs11PanelLayout); - pkcs11PanelLayout.setHorizontalGroup( - pkcs11PanelLayout - .createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addGroup( - pkcs11PanelLayout - .createSequentialGroup() - .addContainerGap() - .addGroup( - pkcs11PanelLayout - .createParallelGroup( - javax.swing.GroupLayout - .Alignment.LEADING) - .addComponent( - pkcs11PasswordField, - javax.swing.GroupLayout - .DEFAULT_SIZE, - 369, - Short.MAX_VALUE) - .addComponent(driverLabel) - .addComponent(passwordPkcs11Label) - .addGroup( - pkcs11PanelLayout - .createSequentialGroup() - .addComponent( - driverComboBox, - 0, - 336, - Short.MAX_VALUE) - .addPreferredGap( - javax.swing - .LayoutStyle - .ComponentPlacement - .RELATED) - .addComponent( - driverButton)) - .addComponent( - usePkcs11ExperimentalSliSupportCheckBox) - .addComponent( - addPkcs11Button, - javax.swing.GroupLayout - .Alignment.TRAILING)) - .addContainerGap())); - pkcs11PanelLayout.setVerticalGroup( - pkcs11PanelLayout - .createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addGroup( - pkcs11PanelLayout - .createSequentialGroup() - .addComponent(driverLabel) - .addPreferredGap( - javax.swing.LayoutStyle.ComponentPlacement - .RELATED) - .addGroup( - pkcs11PanelLayout - .createParallelGroup( - javax.swing.GroupLayout - .Alignment.BASELINE) - .addComponent(driverButton) - .addComponent( - driverComboBox, - javax.swing.GroupLayout - .PREFERRED_SIZE, - 17, - javax.swing.GroupLayout - .PREFERRED_SIZE)) - .addPreferredGap( - javax.swing.LayoutStyle.ComponentPlacement - .RELATED) - .addComponent(passwordPkcs11Label) - .addPreferredGap( - javax.swing.LayoutStyle.ComponentPlacement - .RELATED) - .addComponent( - pkcs11PasswordField, - javax.swing.GroupLayout.PREFERRED_SIZE, - javax.swing.GroupLayout.DEFAULT_SIZE, - javax.swing.GroupLayout.PREFERRED_SIZE) - .addPreferredGap( - javax.swing.LayoutStyle.ComponentPlacement - .RELATED) - .addComponent(usePkcs11ExperimentalSliSupportCheckBox) - .addPreferredGap( - javax.swing.LayoutStyle.ComponentPlacement - .RELATED) - .addComponent(addPkcs11Button) - .addGap(58, 58, 58))); - - pkcs11PanelLayout.linkSize( - javax.swing.SwingConstants.VERTICAL, - new java.awt.Component[] { - addPkcs11Button, driverButton, driverComboBox, pkcs11PasswordField - }); - - certificatejTabbedPane.addTab( - Constant.messages.getString("options.cert.tab.pkcs11"), pkcs11Panel); - - useClientCertificateCheckBox.setText( - Constant.messages.getString("options.cert.label.useclientcert")); - useClientCertificateCheckBox.setBorder( - javax.swing.BorderFactory.createEmptyBorder(0, 0, 0, 0)); - useClientCertificateCheckBox.setMargin(new java.awt.Insets(0, 0, 0, 0)); - useClientCertificateCheckBox.addActionListener( - new java.awt.event.ActionListener() { - @Override - public void actionPerformed(java.awt.event.ActionEvent evt) { - useClientCertificateCheckBoxActionPerformed(evt); - } - }); - - textLabel.setText(Constant.messages.getString("options.cert.label.addkeystore")); - - certificateLabel.setText(Constant.messages.getString("options.cert.label.activecerts")); - - certificateTextField.setEnabled(false); - certificateTextField.setEditable(false); - - showActiveCertificateButton.setText("->"); - showActiveCertificateButton.setActionCommand(">"); - showActiveCertificateButton.setEnabled(false); - showActiveCertificateButton.setMargin(new java.awt.Insets(2, 2, 2, 2)); - showActiveCertificateButton.addActionListener( - new java.awt.event.ActionListener() { - @Override - public void actionPerformed(java.awt.event.ActionEvent evt) { - showActiveCertificateButtonActionPerformed(evt); - } - }); - - javax.swing.GroupLayout certificatePanelLayout = - new javax.swing.GroupLayout(certificatePanel); - certificatePanel.setLayout(certificatePanelLayout); - certificatePanelLayout.setHorizontalGroup( - certificatePanelLayout - .createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addGroup( - certificatePanelLayout - .createParallelGroup( - javax.swing.GroupLayout.Alignment.LEADING) - .addGroup( - certificatePanelLayout - .createSequentialGroup() - .addComponent( - textLabel, - 0, - 0, - Short.MAX_VALUE) - .addContainerGap()) - .addGroup( - certificatePanelLayout - .createSequentialGroup() - .addGap(2, 2, 2) - .addGroup( - certificatePanelLayout - .createParallelGroup( - javax.swing - .GroupLayout - .Alignment - .LEADING) - .addComponent( - certificatejTabbedPane, - javax.swing - .GroupLayout - .DEFAULT_SIZE, - 394, - Short.MAX_VALUE) - .addGroup( - certificatePanelLayout - .createSequentialGroup() - .addGroup( - certificatePanelLayout - .createParallelGroup( - javax - .swing - .GroupLayout - .Alignment - .LEADING) - .addComponent( - useClientCertificateCheckBox) - .addComponent( - certificateLabel) - .addGroup( - javax - .swing - .GroupLayout - .Alignment - .TRAILING, - certificatePanelLayout - .createSequentialGroup() - .addComponent( - certificateTextField, - javax - .swing - .GroupLayout - .DEFAULT_SIZE, - 363, - Short - .MAX_VALUE) - .addPreferredGap( - javax - .swing - .LayoutStyle - .ComponentPlacement - .UNRELATED) - .addComponent( - showActiveCertificateButton) - .addPreferredGap( - javax - .swing - .LayoutStyle - .ComponentPlacement - .RELATED))) - .addContainerGap()))))); - certificatePanelLayout.setVerticalGroup( - certificatePanelLayout - .createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addGroup( - certificatePanelLayout - .createSequentialGroup() - .addContainerGap() - .addComponent(textLabel) - .addPreferredGap( - javax.swing.LayoutStyle.ComponentPlacement - .UNRELATED) - .addComponent(useClientCertificateCheckBox) - .addPreferredGap( - javax.swing.LayoutStyle.ComponentPlacement - .RELATED) - .addComponent( - certificatejTabbedPane, - javax.swing.GroupLayout.DEFAULT_SIZE, - 152, - Short.MAX_VALUE) - .addPreferredGap( - javax.swing.LayoutStyle.ComponentPlacement - .RELATED) - .addComponent(certificateLabel) - .addPreferredGap( - javax.swing.LayoutStyle.ComponentPlacement - .RELATED) - .addGroup( - certificatePanelLayout - .createParallelGroup( - javax.swing.GroupLayout - .Alignment.BASELINE) - .addComponent( - certificateTextField, - javax.swing.GroupLayout - .PREFERRED_SIZE, - javax.swing.GroupLayout - .DEFAULT_SIZE, - javax.swing.GroupLayout - .PREFERRED_SIZE) - .addComponent( - showActiveCertificateButton, - javax.swing.GroupLayout - .DEFAULT_SIZE, - javax.swing.GroupLayout - .DEFAULT_SIZE, - Short.MAX_VALUE)) - .addContainerGap())); - - certificatePanelLayout.linkSize( - javax.swing.SwingConstants.VERTICAL, - new java.awt.Component[] {certificateTextField, showActiveCertificateButton}); - - javax.swing.GroupLayout layout = new javax.swing.GroupLayout(getContentPane()); - getContentPane().setLayout(layout); - layout.setHorizontalGroup( - layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent( - certificatePanel, - javax.swing.GroupLayout.DEFAULT_SIZE, - javax.swing.GroupLayout.DEFAULT_SIZE, - Short.MAX_VALUE)); - layout.setVerticalGroup( - layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addGroup( - layout.createSequentialGroup() - .addComponent( - certificatePanel, - javax.swing.GroupLayout.DEFAULT_SIZE, - javax.swing.GroupLayout.DEFAULT_SIZE, - Short.MAX_VALUE) - .addContainerGap())); - - // ************************************************************************** - // end netbeans code - // ************************************************************************** - } - return certificatePanel; - } - - private static void showKeyStoreCertError(String errorMessage) { - showCertError("options.cert.error.accesskeystore", errorMessage); - } - - private static void showCertError(String i18nKeyBaseMessage, String errorMessage) { - JOptionPane.showMessageDialog( - null, - new String[] {Constant.messages.getString(i18nKeyBaseMessage), errorMessage}, - Constant.messages.getString("options.cert.error"), - JOptionPane.ERROR_MESSAGE); - } - - private void keyStoreListSelectionChanged() { - int keystore = keyStoreList.getSelectedIndex(); - try { - aliasTableModel.setKeystore(keystore); - } catch (Exception e) { - showKeyStoreCertError(e.toString()); - logger.error(e.getMessage(), e); - } - } - - private void showActiveCertificateButtonActionPerformed( - java.awt.event.ActionEvent - evt) { // GEN-FIRST:event_showActiveCertificateButtonActionPerformed - Certificate cert = contextManager.getDefaultCertificate(); - if (cert != null) { - showCertificate(cert); - } - } // GEN-LAST:event_showActiveCertificateButtonActionPerformed - - private void addPkcs11ButtonActionPerformed( - java.awt.event.ActionEvent evt) { // GEN-FIRST:event_addPkcs11ButtonActionPerformed - String name = null; - try { - final int indexSelectedDriver = driverComboBox.getSelectedIndex(); - name = driverConfig.getNames().get(indexSelectedDriver); - if (name.equals("")) { - return; - } - - String library = driverConfig.getPaths().get(indexSelectedDriver); - if (library.equals("")) { - return; - } - - int slot = driverConfig.getSlots().get(indexSelectedDriver); - if (slot < 0) { - return; - } - - int slotListIndex = driverConfig.getSlotIndexes().get(indexSelectedDriver); - if (slotListIndex < 0) { - return; - } - - String kspass = new String(pkcs11PasswordField.getPassword()); - if (kspass.equals("")) { - kspass = null; - } - - ch.csnc.extension.httpclient.PKCS11Configuration.PCKS11ConfigurationBuilder - confBuilder = ch.csnc.extension.httpclient.PKCS11Configuration.builder(); - confBuilder.setName(name).setLibrary(library); - if (usePkcs11ExperimentalSliSupportCheckBox.isSelected()) { - confBuilder.setSlotListIndex(slotListIndex); - } else { - confBuilder.setSlotId(slot); - } - - int ksIndex = contextManager.initPKCS11(confBuilder.build(), kspass); - - if (ksIndex == -1) { - logger.error( - "The required PKCS#11 provider is not available (" - + ch.csnc.extension.httpclient.SSLContextManager - .SUN_PKCS11_CANONICAL_CLASS_NAME - + " or " - + ch.csnc.extension.httpclient.SSLContextManager - .IBM_PKCS11_CANONICAL_CLASS_NAME - + ")."); - showErrorMessageSunPkcs11ProviderNotAvailable(); - return; - } - - // The PCKS11 driver/smartcard was initialized properly: reset login attempts - login_attempts = 0; - keyStoreListModel.insertElementAt( - contextManager.getKeyStoreDescription(ksIndex), ksIndex); - // Issue 182 - retry = true; - - certificatejTabbedPane.setSelectedIndex(0); - activateFirstOnlyAliasOfKeyStore(ksIndex); - - driverComboBox.setSelectedIndex(-1); - pkcs11PasswordField.setText(""); - - } catch (InvocationTargetException e) { - if (e.getCause() instanceof ProviderException) { - if ("Error parsing configuration".equals(e.getCause().getMessage())) { - // There was a problem with the configuration provided: - // - Missing library. - // - Malformed configuration. - // - ... - logAndShowGenericErrorMessagePkcs11CouldNotBeAdded(false, name, e); - } else if ("Initialization failed".equals(e.getCause().getMessage())) { - // The initialisation may fail because of: - // - no smart card reader or smart card detected. - // - smart card is in use by other application. - // - ... - - // Issue 182: Try to instantiate the PKCS11 provider twice if there are - // conflicts with other software (e.g.. Firefox), that is accessing it too. - if (retry) { - // Try two times only - retry = false; - addPkcs11ButtonActionPerformed(evt); - } else { - JOptionPane.showMessageDialog( - null, - new String[] { - Constant.messages.getString("options.cert.error"), - Constant.messages.getString("options.cert.error.pkcs11") - }, - Constant.messages.getString("options.cert.label.client.cert"), - JOptionPane.ERROR_MESSAGE); - // Error message changed to explain that user should try to add it again... - retry = true; - logger.warn("Couldn't add key from {}", name, e); - } - } else { - logAndShowGenericErrorMessagePkcs11CouldNotBeAdded(false, name, e); - } - } else { - logAndShowGenericErrorMessagePkcs11CouldNotBeAdded(false, name, e); - } - } catch (java.io.IOException e) { - if (e.getMessage().equals("load failed") - && e.getCause() - .getClass() - .getName() - .equals("javax.security.auth.login.FailedLoginException")) { - // Exception due to a failed login attempt: BAD PIN or password - login_attempts++; - String attempts = " (" + login_attempts + "/" + MAX_LOGIN_ATTEMPTS + ") "; - if (login_attempts == (MAX_LOGIN_ATTEMPTS - 1)) { - // Last attempt before blocking the smartcard - JOptionPane.showMessageDialog( - null, - new String[] { - Constant.messages.getString("options.cert.error"), - Constant.messages.getString("options.cert.error.wrongpassword"), - Constant.messages.getString("options.cert.error.wrongpasswordlast"), - attempts - }, - Constant.messages.getString("options.cert.label.client.cert"), - JOptionPane.ERROR_MESSAGE); - logger.warn( - "PKCS#11: Incorrect PIN or password {}: {} *LAST TRY BEFORE BLOCKING*", - attempts, - name); - } else { - JOptionPane.showMessageDialog( - null, - new String[] { - Constant.messages.getString("options.cert.error"), - Constant.messages.getString("options.cert.error.wrongpassword"), - attempts - }, - Constant.messages.getString("options.cert.label.client.cert"), - JOptionPane.ERROR_MESSAGE); - logger.warn("PKCS#11: Incorrect PIN or password {}:{}", attempts, name); - } - } else { - logAndShowGenericErrorMessagePkcs11CouldNotBeAdded(false, name, e); - } - } catch (KeyStoreException e) { - logAndShowGenericErrorMessagePkcs11CouldNotBeAdded(false, name, e); - } catch (Exception e) { - logAndShowGenericErrorMessagePkcs11CouldNotBeAdded(true, name, e); - } - } // GEN-LAST:event_addPkcs11ButtonActionPerformed - - private void activateFirstOnlyAliasOfKeyStore(int ksIndex) { - if (ksIndex < 0 || ksIndex >= keyStoreList.getModel().getSize()) { - return; - } - - keyStoreList.setSelectedIndex(ksIndex); - if (aliasTable.getRowCount() != 0) { - aliasTable.setRowSelectionInterval(0, 0); - - if (aliasTable.getRowCount() == 1 && !isCertActive()) { - setActiveAction(); - } - } - } - - private boolean isCertActive() { - String currentKey = contextManager.getDefaultKey(); - return currentKey != null && !currentKey.isEmpty(); - } - - private void showErrorMessageSunPkcs11ProviderNotAvailable() { - final String sunReference = - Constant.messages.getString("options.cert.error.pkcs11notavailable.sun.hyperlink"); - final String ibmReference = - Constant.messages.getString("options.cert.error.pkcs11notavailable.ibm.hyperlink"); - Object[] hyperlinks = new Object[2]; - try { - JXHyperlink hyperlinkLabel = new JXHyperlink(); - hyperlinkLabel.setURI(URI.create(sunReference)); - hyperlinkLabel.setText( - Constant.messages.getString( - "options.cert.error.pkcs11notavailable.sun.hyperlink.text")); - hyperlinks[0] = hyperlinkLabel; - - hyperlinkLabel = new JXHyperlink(); - hyperlinkLabel.setURI(URI.create(ibmReference)); - hyperlinkLabel.setText( - Constant.messages.getString( - "options.cert.error.pkcs11notavailable.ibm.hyperlink.text")); - hyperlinks[1] = hyperlinkLabel; - } catch (UnsupportedOperationException e) { - // Show plain text instead of a hyperlink if the current platform doesn't support - // Desktop. - hyperlinks[0] = sunReference; - hyperlinks[1] = ibmReference; - } - - JOptionPane.showMessageDialog( - null, - new Object[] { - Constant.messages.getString("options.cert.error"), - Constant.messages.getString("options.cert.error.pkcs11notavailable"), - hyperlinks - }, - Constant.messages.getString("options.cert.label.client.cert"), - JOptionPane.ERROR_MESSAGE); - } - - private void logAndShowGenericErrorMessagePkcs11CouldNotBeAdded( - boolean isErrorLevel, String name, Exception e) { - if (pkcs11PasswordField.getPassword().length == 0) { - JOptionPane.showMessageDialog( - null, - new String[] { - Constant.messages.getString("options.cert.error"), - Constant.messages.getString("options.cert.error.password.blank") - }, - Constant.messages.getString("options.cert.label.client.cert"), - JOptionPane.ERROR_MESSAGE); - } else { - JOptionPane.showMessageDialog( - null, - new String[] { - Constant.messages.getString("options.cert.error"), - Constant.messages.getString("options.cert.error.password") - }, - Constant.messages.getString("options.cert.label.client.cert"), - JOptionPane.ERROR_MESSAGE); - if (isErrorLevel) { - logger.error("Couldn't add key from {}", name, e); - } else { - logger.warn("Couldn't add key from {}", name, e); - } - } - } - - private void driverButtonActionPerformed( - java.awt.event.ActionEvent evt) { // GEN-FIRST:event_driverButtonActionPerformed - new JDialog(new ch.csnc.extension.ui.DriversView(driverConfig), true); - } // GEN-LAST:event_driverButtonActionPerformed - - private void addPkcs12ButtonActionPerformed( - java.awt.event.ActionEvent evt) { // GEN-FIRST:event_addPkcs12ButtonActionPerformed - if (fileTextField.getText().equals("")) { - return; - } - String kspass = new String(pkcs12PasswordField.getPassword()); - if (kspass.equals("")) { - // pcks#12 file with empty password is not supported by java - showCertError( - "options.cert.error.pkcs12nopass", - Constant.messages.getString("options.cert.error.usepassfile")); - return; - } - - int ksIndex; - try { - ksIndex = contextManager.loadPKCS12Certificate(fileTextField.getText(), kspass); - keyStoreListModel.insertElementAt( - contextManager.getKeyStoreDescription(ksIndex), ksIndex); - } catch (Exception e) { - showKeyStoreCertError(Constant.messages.getString("options.cert.error.password")); - logger.error(e.getMessage(), e); - return; - } - - certificatejTabbedPane.setSelectedIndex(0); - activateFirstOnlyAliasOfKeyStore(ksIndex); - - fileTextField.setText(""); - pkcs12PasswordField.setText(""); - } // GEN-LAST:event_addPkcs12ButtonActionPerformed - - private void browseButtonActionPerformed( - java.awt.event.ActionEvent evt) { // GEN-FIRST:event_browseButtonActionPerformed - JFileChooser fc = new JFileChooser(); - fc.setFileFilter( - new FileNameExtensionFilter( - Constant.messages.getString("options.cert.label.client.cert") - + " (*.p12, *.pfx)", - "p12", - "pfx")); - - int state = fc.showOpenDialog(null); - - if (state == JFileChooser.APPROVE_OPTION) { - fileTextField.setText(fc.getSelectedFile().toString()); - } - } // GEN-LAST:event_browseButtonActionPerformed - - private void showAliasButtonActionPerformed( - java.awt.event.ActionEvent evt) { // GEN-FIRST:event_showAliasButtonActionPerformed - int keystore = keyStoreList.getSelectedIndex(); - if (keystore >= 0) { - int alias = aliasTable.getSelectedRow(); - Certificate cert = contextManager.getCertificate(keystore, alias); - if (cert != null) { - showCertificate(cert); - } - } - } // GEN-LAST:event_showAliasButtonActionPerformed - - /** - * Shows a second {@link JFrame} displaying the content of the certificate - * - * @param cert - */ - @SuppressWarnings("unused") - private void showCertificate(Certificate cert) { - if (cert != null) { - new ch.csnc.extension.ui.CertificateView(cert.toString()); - } - } - - private void setActiveButtonActionPerformed( - java.awt.event.ActionEvent evt) { // GEN-FIRST:event_setActiveButtonActionPerformed - setActiveAction(); - } // GEN-LAST:event_setActiveButtonActionPerformed - - private void setActiveAction() { - int ks = keyStoreList.getSelectedIndex(); - int alias = aliasTable.getSelectedRow(); - if (ks > -1 && alias > -1) { - if (!contextManager.isKeyUnlocked(ks, alias)) { - - try { - if (!contextManager.unlockKeyWithDefaultPassword(ks, alias)) { - String password = getPassword(); - - if (!contextManager.unlockKey(ks, alias, password)) { - JOptionPane.showMessageDialog( - null, - new String[] { - Constant.messages.getString( - "options.cert.error.accesskeystore") - }, - Constant.messages.getString("options.cert.error"), - JOptionPane.ERROR_MESSAGE); - } - } - } catch (Exception e) { - showKeyStoreCertError(e.toString()); - } - } - Certificate cert = contextManager.getCertificate(ks, alias); - try { - contextManager.getFingerPrint(cert); - } catch (KeyStoreException kse) { - showCertError("options.cert.error.fingerprint", kse.toString()); - } - - try { - contextManager.setDefaultKey(ks, alias); - - OptionsParamCertificate certParam = - Model.getSingleton().getOptionsParam().getCertificateParam(); - certParam.setActiveCertificate(); - - } catch (KeyStoreException e) { - logger.error(e.getMessage(), e); - } - certificateTextField.setText(contextManager.getDefaultKey()); - } - } - - public String getPassword() { - JPasswordField askPasswordField = new JPasswordField(); - int result = - JOptionPane.showConfirmDialog( - this, - askPasswordField, - Constant.messages.getString("options.cert.label.enterpassword"), - JOptionPane.OK_CANCEL_OPTION); - if (result == JOptionPane.OK_OPTION) { - return new String(askPasswordField.getPassword()); - } else return null; - } - - private void deleteButtonActionPerformed( - java.awt.event.ActionEvent evt) { // GEN-FIRST:event_deleteButtonActionPerformed - int index = keyStoreList.getSelectedIndex(); - if (index >= 0) { - boolean isDefaultKeyStore = contextManager.removeKeyStore(index); - if (isDefaultKeyStore) { - certificateTextField.setText(""); - } - keyStoreListModel.removeElementAt(keyStoreList.getSelectedIndex()); - aliasTableModel.removeKeystore(); - } - } // GEN-LAST:event_deleteButtonActionPerformed - - private void useClientCertificateCheckBoxActionPerformed( - java.awt.event.ActionEvent - evt) { // GEN-FIRST:event_useClientCertificateCheckBoxActionPerformed - // The enable unsafe SSL renegotiation checkbox is independent of using a client certificate - // (although commonly related) - // enableUnsafeSSLRenegotiationCheckBox.setEnabled(useClientCertificateCheckBox.isSelected()); - - // keyStore tab - certificatejTabbedPane.setEnabled(useClientCertificateCheckBox.isSelected()); - - keyStoreScrollPane.setEnabled(useClientCertificateCheckBox.isSelected()); - keyStoreList.setEnabled(useClientCertificateCheckBox.isSelected()); - - aliasScrollPane.setEnabled(useClientCertificateCheckBox.isSelected()); - aliasTable.setEnabled(useClientCertificateCheckBox.isSelected()); - - deleteButton.setEnabled(useClientCertificateCheckBox.isSelected()); - setActiveButton.setEnabled(useClientCertificateCheckBox.isSelected()); - showAliasButton.setEnabled(useClientCertificateCheckBox.isSelected()); - - // pkcs12 tab - fileTextField.setEnabled(useClientCertificateCheckBox.isSelected()); - browseButton.setEnabled(useClientCertificateCheckBox.isSelected()); - pkcs12PasswordField.setEnabled(useClientCertificateCheckBox.isSelected()); - addPkcs12Button.setEnabled(useClientCertificateCheckBox.isSelected()); - - // pkcs11 tab - driverComboBox.setEnabled(useClientCertificateCheckBox.isSelected()); - driverButton.setEnabled(useClientCertificateCheckBox.isSelected()); - pkcs11PasswordField.setEnabled(useClientCertificateCheckBox.isSelected()); - addPkcs11Button.setEnabled(useClientCertificateCheckBox.isSelected()); - usePkcs11ExperimentalSliSupportCheckBox.setEnabled( - useClientCertificateCheckBox.isSelected()); - usePkcs11ExperimentalSliSupportCheckBox.setSelected( - Model.getSingleton() - .getOptionsParam() - .getExperimentalFeaturesParam() - .isExperimentalSliSupportEnabled()); - - // actual certificate fields - certificateTextField.setEnabled(useClientCertificateCheckBox.isSelected()); - showActiveCertificateButton.setEnabled(useClientCertificateCheckBox.isSelected()); - } // GEN-LAST:event_useClientCertificateCheckBoxActionPerformed - - private void usePkcs11ExperimentalSliSupportCheckBoxActionPerformed( - java.awt.event.ActionEvent evt) { - Model.getSingleton() - .getOptionsParam() - .getExperimentalFeaturesParam() - .setSlotListIndexSupport(usePkcs11ExperimentalSliSupportCheckBox.isSelected()); - } - - // TODO remove - private OptionsCertificatePanel getContentPane() { - return this; - } - - @Override - public void initParam(Object obj) { - OptionsParam options = (OptionsParam) obj; - OptionsParamCertificate certParam = options.getCertificateParam(); - - // Should only run once after startup if client certificate is set from commandline - if (overrideEnableClientCertificate) { - certParam.setEnableCertificate(true); - overrideEnableClientCertificate = false; - } - keyStoreListModel.clear(); - for (int i = 0; i < contextManager.getKeyStoreCount(); i++) { - keyStoreListModel.addElement(contextManager.getKeyStoreDescription(i)); - } - Certificate cert = contextManager.getDefaultCertificate(); - if (cert != null) { - certificateTextField.setText(cert.toString()); - } - useClientCertificateCheckBox.setSelected(certParam.isUseClientCert()); - useClientCertificateCheckBoxActionPerformed(null); - - // getBtnLocation().setEnabled(getChkUseClientCertificate().isSelected()); - // getTxtLocation().setText(options.getCertificateParam().getClientCertLocation()); - } - - @Override - public void saveParam(Object obj) throws Exception { - OptionsParam options = (OptionsParam) obj; - OptionsParamCertificate certParam = options.getCertificateParam(); - certParam.setEnableCertificate(useClientCertificateCheckBox.isSelected()); - } - - @Override - public String getHelpIndex() { - // ZAP: added help index - return "ui.dialogs.options.certificate"; - } -} // @jve:decl-index=0:visual-constraint="10,10" diff --git a/zap/src/main/java/org/parosproxy/paros/extension/option/OptionsParamCertificate.java b/zap/src/main/java/org/parosproxy/paros/extension/option/OptionsParamCertificate.java deleted file mode 100644 index a73078f1604..00000000000 --- a/zap/src/main/java/org/parosproxy/paros/extension/option/OptionsParamCertificate.java +++ /dev/null @@ -1,244 +0,0 @@ -/* - * - * Paros and its related class files. - * - * Paros is an HTTP/HTTPS proxy for assessing web application security. - * Copyright (C) 2003-2004 Chinotec Technologies Company - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the Clarified Artistic License - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Clarified Artistic License for more details. - * - * You should have received a copy of the Clarified Artistic License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - */ -// ZAP: 2012/04/25 Added @Override annotation to the appropriate method. -// ZAP: 2013/01/25 Removed the "(non-Javadoc)" comments. -// ZAP: 2013/03/03 Issue 546: Remove all template Javadoc comments -// ZAP: 2014/03/23 Issue 412: Enable unsafe SSL/TLS renegotiation option not saved -// ZAP: 2014/08/14 Issue 1184: Improve support for IBM JDK -// ZAP: 2017/09/26 Use helper methods to read the configurations. -// ZAP: 2018/02/14 Remove unnecessary boxing / unboxing -// ZAP: 2018/08/01 Added support for setting and persisting client cert from CLI -// ZAP: 2019/06/01 Normalise line endings. -// ZAP: 2019/06/05 Normalise format/style. -// ZAP: 2020/11/26 Use Log4j 2 classes for logging. -// ZAP: 2022/05/21 Disable unsafe SSL/TLS renegotiation option. -// ZAP: 2022/05/29 Deprecate the class. -// ZAP: 2022/06/07 Address deprecation warnings with SSLConnector. -package org.parosproxy.paros.extension.option; - -import java.io.File; -import java.io.IOException; -import java.security.KeyManagementException; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; -import java.security.cert.CertificateException; -import org.apache.commons.httpclient.protocol.Protocol; -import org.apache.commons.httpclient.protocol.ProtocolSocketFactory; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.parosproxy.paros.common.AbstractParam; - -/** - * @deprecated (2.12.0) No longer in use. - */ -@Deprecated -public class OptionsParamCertificate extends AbstractParam { - - private static final Logger logger = LogManager.getLogger(OptionsParamCertificate.class); - - private static final String CERTIFICATE_BASE_KEY = "certificate"; - - private static final String USE_CLIENT_CERT = CERTIFICATE_BASE_KEY + ".use"; - private static final String PERSIST_CLIENT_CERT = CERTIFICATE_BASE_KEY + ".persist"; - private static final String CLIENT_CERT_LOCATION = CERTIFICATE_BASE_KEY + ".pkcs12.path"; - private static final String CLIENT_CERT_PASSWORD = CERTIFICATE_BASE_KEY + ".pkcs12.password"; - private static final String CLIENT_CERT_INDEX = CERTIFICATE_BASE_KEY + ".pkcs12.index"; - - private boolean useClientCert = false; - private String clientCertLocation = ""; - private String clientCertPassword = ""; - private int clientCertIndex = 0; - - public OptionsParamCertificate() {} - - @Override - protected void parse() { - - clientCertCheck(); - saveClientCertSettings(); - } - - /** - * Saves the client cert settings if the flag is set explicitly. Only works for the CLI - * currently. - */ - private void saveClientCertSettings() { - - if (getBoolean(PERSIST_CLIENT_CERT, false)) { - logger.warn("Saving Client Certificate settings: password will be found in config"); - setUseClientCert(getBoolean(USE_CLIENT_CERT, false)); - setClientCertLocation(getString(CLIENT_CERT_LOCATION, "")); - setClientCertPassword(getString(CLIENT_CERT_PASSWORD, "")); - setClientCertIndex(getInt(CLIENT_CERT_INDEX, 0)); - - } else { - // Default to clear settings - setUseClientCert(false); - setClientCertLocation(""); - setClientCertPassword(""); - setClientCertIndex(0); - } - } - - /** - * Enables ClientCertificate from -config CLI parameters Requires location, password and a flag - * to use client certificate. - */ - private void clientCertCheck() { - - boolean enableClientCert = getBoolean(USE_CLIENT_CERT, false); - String certPath = getString(CLIENT_CERT_LOCATION, ""); - String certPass = getString(CLIENT_CERT_PASSWORD, ""); - int certIndex = getInt(CLIENT_CERT_INDEX, 0); - - if (enableClientCert && !certPath.isEmpty() && !certPass.isEmpty()) { - try { - - ch.csnc.extension.httpclient.SSLContextManager contextManager = - getSSLContextManager(); - int ksIndex = contextManager.loadPKCS12Certificate(certPath, certPass); - contextManager.unlockKey(ksIndex, certIndex, certPass); - contextManager.setDefaultKey(ksIndex, certIndex); - - setActiveCertificate(); - setEnableCertificate(true); - - logger.info("Client Certificate enabled from CLI"); - logger.info("Use -config certificate.persist=true to save settings"); - - } catch (IOException - | CertificateException - | NoSuchAlgorithmException - | KeyStoreException - | KeyManagementException ex) { - logger.error("The certificate could not be enabled due to an error", ex); - } - } - } - - public String getClientCertPassword() { - return clientCertPassword; - } - - public void setClientCertPassword(String clientCertPassword) { - this.clientCertPassword = clientCertPassword; - getConfig().setProperty(CLIENT_CERT_PASSWORD, clientCertPassword); - } - - /** - * @return Returns the client cert location. - */ - public String getClientCertLocation() { - return clientCertLocation; - } - - public void setClientCertLocation(String clientCertLocation) { - if (clientCertLocation != null && !clientCertLocation.equals("")) { - File file = new File(clientCertLocation); - if (!file.exists()) { - setUseClientCert(false); - return; - } - } else { - setUseClientCert(false); - } - this.clientCertLocation = clientCertLocation; - getConfig().setProperty(CLIENT_CERT_LOCATION, clientCertLocation); - } - - public int getClientCertIndex() { - return clientCertIndex; - } - - public void setClientCertIndex(int clientCertIdx) { - this.clientCertIndex = clientCertIdx; - getConfig().setProperty(CLIENT_CERT_INDEX, Integer.toString(clientCertIndex)); - } - - public boolean isUseClientCert() { - return useClientCert; - } - - private void setUseClientCert(boolean isUse) { - useClientCert = isUse; - getConfig().setProperty(USE_CLIENT_CERT, Boolean.toString(useClientCert)); - } - - public void setEnableCertificate(boolean enabled) { - ProtocolSocketFactory sslFactory = Protocol.getProtocol("https").getSocketFactory(); - - if (sslFactory instanceof org.parosproxy.paros.network.SSLConnector) { - org.parosproxy.paros.network.SSLConnector ssl = - (org.parosproxy.paros.network.SSLConnector) sslFactory; - ssl.setEnableClientCert(enabled); - - setUseClientCert(enabled); - } - } - - public void setActiveCertificate() { - - ProtocolSocketFactory sslFactory = Protocol.getProtocol("https").getSocketFactory(); - - if (sslFactory instanceof org.parosproxy.paros.network.SSLConnector) { - org.parosproxy.paros.network.SSLConnector ssl = - (org.parosproxy.paros.network.SSLConnector) sslFactory; - ssl.setActiveCertificate(); - } - } - - public ch.csnc.extension.httpclient.SSLContextManager getSSLContextManager() { - - ProtocolSocketFactory sslFactory = Protocol.getProtocol("https").getSocketFactory(); - if (sslFactory instanceof org.parosproxy.paros.network.SSLConnector) { - org.parosproxy.paros.network.SSLConnector ssl = - (org.parosproxy.paros.network.SSLConnector) sslFactory; - - return ssl.getSSLContextManager(); - } - return null; - } - - /** - * Tells whether or not the unsafe SSL renegotiation is enabled. - * - * @return {@code true} if the unsafe SSL renegotiation is enabled, {@code false} otherwise. - * @see #setAllowUnsafeSslRenegotiation(boolean) - */ - public boolean isAllowUnsafeSslRenegotiation() { - return false; - } - - /** - * Sets whether or not the unsafe SSL renegotiation is enabled. - * - *

Calling this method changes the system property - * "sun.security.ssl.allowUnsafeRenegotiation" and "com.ibm.jsse2.renegotiate". It must be set - * before establishing any SSL connection. Further changes after establishing a SSL connection - * will have no effect on the renegotiation but the value will be saved and restored next time - * ZAP is restarted. - * - * @param allow {@code true} if the unsafe SSL renegotiation should be enabled, {@code false} - * otherwise. - * @see #isAllowUnsafeSslRenegotiation() - */ - public void setAllowUnsafeSslRenegotiation(boolean allow) {} -} diff --git a/zap/src/main/java/org/parosproxy/paros/model/OptionsParam.java b/zap/src/main/java/org/parosproxy/paros/model/OptionsParam.java index 571d2a5b3ea..7fc27f8b972 100644 --- a/zap/src/main/java/org/parosproxy/paros/model/OptionsParam.java +++ b/zap/src/main/java/org/parosproxy/paros/model/OptionsParam.java @@ -85,10 +85,6 @@ public class OptionsParam extends AbstractParam { private OptionsParamView viewParam = new OptionsParamView(); - @SuppressWarnings("deprecation") - private org.parosproxy.paros.extension.option.OptionsParamCertificate certificateParam = - new org.parosproxy.paros.extension.option.OptionsParamCertificate(); - // ZAP: Added many instance variables for new functionality. private OptionsParamCheckForUpdates checkForUpdatesParam = new OptionsParamCheckForUpdates(); private OptionsParamApi apiParam = new OptionsParamApi(); @@ -103,10 +99,6 @@ public List getTokensNames() { } }; - @SuppressWarnings("deprecation") - private ch.csnc.extension.util.OptionsParamExperimentalSliSupport experimentalFeaturesParam = - new ch.csnc.extension.util.OptionsParamExperimentalSliSupport(); - /** The database configurations. */ // ZAP: Added the instance variable. private DatabaseParam databaseParam = new DatabaseParam(); @@ -177,25 +169,6 @@ public OptionsParamCheckForUpdates getCheckForUpdatesParam() { return checkForUpdatesParam; } - /** - * @param certificateParam The certificateParam to set. - * @deprecated (2.12.0) - */ - @Deprecated - public void setCertificateParam( - org.parosproxy.paros.extension.option.OptionsParamCertificate certificateParam) { - this.certificateParam = certificateParam; - } - - /** - * @return Returns the certificateParam. - * @deprecated (2.12.0) - */ - @Deprecated - public org.parosproxy.paros.extension.option.OptionsParamCertificate getCertificateParam() { - return certificateParam; - } - public void addParamSet(AbstractParam paramSet) { paramSetList.add(paramSet); abstractParamsMap.put(paramSet.getClass(), paramSet); @@ -315,15 +288,6 @@ public OptionsParamApi getApiParam() { return apiParam; } - /** - * @deprecated (2.12.0) - */ - @Deprecated - public ch.csnc.extension.util.OptionsParamExperimentalSliSupport - getExperimentalFeaturesParam() { - return experimentalFeaturesParam; - } - /** * Gets the database configurations. * diff --git a/zap/src/main/java/org/parosproxy/paros/network/SSLConnector.java b/zap/src/main/java/org/parosproxy/paros/network/SSLConnector.java index b24dd91d6aa..63850cbe0ca 100644 --- a/zap/src/main/java/org/parosproxy/paros/network/SSLConnector.java +++ b/zap/src/main/java/org/parosproxy/paros/network/SSLConnector.java @@ -194,9 +194,6 @@ public class SSLConnector @SuppressWarnings("deprecation") private static org.parosproxy.paros.security.SslCertificateService sslCertificateService; - @SuppressWarnings("deprecation") - private static ch.csnc.extension.httpclient.SSLContextManager sslContextManager = null; - /* * If relaxedTrust then we ignore all of the 'usual' https checks. * This is needed in order to test sites with custom certs @@ -218,39 +215,6 @@ public SSLConnector(boolean relaxedTrust) { clientSSLSockFactory = getClientSocketFactory(SSL); misconfiguredHosts = new LRUMap(10); } - // ZAP: removed ServerSocketFaktory - if (sslContextManager == null) { - sslContextManager = new ch.csnc.extension.httpclient.SSLContextManager(); - } - } - - @SuppressWarnings("deprecation") - public ch.csnc.extension.httpclient.SSLContextManager getSSLContextManager() { - return sslContextManager; - } - - @SuppressWarnings("deprecation") - public void setEnableClientCert(boolean enabled) { - if (enabled) { - if (clientSSLSockCertFactory == null) { - return; - } - - clientSSLSockFactory = clientSSLSockCertFactory; - logger.info("ClientCert enabled using: {}", sslContextManager.getDefaultKey()); - } else { - clientSSLSockFactory = getClientSocketFactory(SSL); - logger.info("ClientCert disabled"); - } - } - - @SuppressWarnings("deprecation") - public void setActiveCertificate() { - - SSLContext sslcont = sslContextManager.getSSLContext(sslContextManager.getDefaultKey()); - clientSSLSockCertFactory = - createDecoratedClientSslSocketFactory(sslcont.getSocketFactory()); - logger.info("ActiveCertificate set to: {}", sslContextManager.getDefaultKey()); } // ZAP: removed server socket methods diff --git a/zap/src/main/java/org/zaproxy/zap/extension/spider/DialogAddDomainAlwaysInScope.java b/zap/src/main/java/org/zaproxy/zap/extension/spider/DialogAddDomainAlwaysInScope.java deleted file mode 100644 index e375445bd46..00000000000 --- a/zap/src/main/java/org/zaproxy/zap/extension/spider/DialogAddDomainAlwaysInScope.java +++ /dev/null @@ -1,240 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2014 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.extension.spider; - -import java.awt.Dialog; -import java.util.regex.Pattern; -import javax.swing.GroupLayout; -import javax.swing.JCheckBox; -import javax.swing.JLabel; -import javax.swing.JOptionPane; -import javax.swing.JPanel; -import javax.swing.event.DocumentEvent; -import javax.swing.event.DocumentListener; -import org.parosproxy.paros.Constant; -import org.zaproxy.zap.utils.ZapTextField; -import org.zaproxy.zap.view.AbstractFormDialog; - -@SuppressWarnings("serial") -/** - * @deprecated (2.12.0) See the spider add-on in zap-extensions instead. - */ -@Deprecated -class DialogAddDomainAlwaysInScope extends AbstractFormDialog { - - private static final long serialVersionUID = -7356390753317082681L; - - private static final String DIALOG_TITLE = - Constant.messages.getString("spider.options.domains.in.scope.add.title"); - - private static final String CONFIRM_BUTTON_LABEL = - Constant.messages.getString("spider.options.domains.in.scope.add.button.confirm"); - - private static final String DOMAIN_FIELD_LABEL = - Constant.messages.getString("spider.options.domains.in.scope.field.label.domain"); - private static final String REGEX_FIELD_LABEL = - Constant.messages.getString("spider.options.domains.in.scope.field.label.regex"); - private static final String ENABLED_FIELD_LABEL = - Constant.messages.getString("spider.options.domains.in.scope.field.label.enabled"); - - private static final String TITLE_INVALID_REGEX_DIALOG = - Constant.messages.getString( - "spider.options.domains.in.scope.warning.invalid.regex.title"); - private static final String TEXT_INVALID_REGEX_DIALOG = - Constant.messages.getString( - "spider.options.domains.in.scope.warning.invalid.regex.text"); - - private ZapTextField domainTextField; - private JCheckBox regexCheckBox; - private JCheckBox enabledCheckBox; - - protected org.zaproxy.zap.spider.DomainAlwaysInScopeMatcher domainAlwaysInScope; - - private ConfirmButtonValidatorDocListener confirmButtonValidatorDocListener; - - public DialogAddDomainAlwaysInScope(Dialog owner) { - super(owner, DIALOG_TITLE); - } - - protected DialogAddDomainAlwaysInScope(Dialog owner, String title) { - super(owner, title); - } - - @Override - protected JPanel getFieldsPanel() { - JPanel fieldsPanel = new JPanel(); - - GroupLayout layout = new GroupLayout(fieldsPanel); - fieldsPanel.setLayout(layout); - layout.setAutoCreateGaps(true); - layout.setAutoCreateContainerGaps(true); - - JLabel domainLabel = new JLabel(DOMAIN_FIELD_LABEL); - JLabel regexLabel = new JLabel(REGEX_FIELD_LABEL); - JLabel enabledLabel = new JLabel(ENABLED_FIELD_LABEL); - - layout.setHorizontalGroup( - layout.createSequentialGroup() - .addGroup( - layout.createParallelGroup(GroupLayout.Alignment.TRAILING) - .addComponent(domainLabel) - .addComponent(enabledLabel) - .addComponent(regexLabel)) - .addGroup( - layout.createParallelGroup(GroupLayout.Alignment.LEADING) - .addComponent(getDomainTextField()) - .addComponent(getEnabledCheckBox()) - .addComponent(getRegexCheckBox()))); - - layout.setVerticalGroup( - layout.createSequentialGroup() - .addGroup( - layout.createParallelGroup(GroupLayout.Alignment.BASELINE) - .addComponent(domainLabel) - .addComponent(getDomainTextField())) - .addGroup( - layout.createParallelGroup(GroupLayout.Alignment.BASELINE) - .addComponent(regexLabel) - .addComponent(getRegexCheckBox())) - .addGroup( - layout.createParallelGroup(GroupLayout.Alignment.BASELINE) - .addComponent(enabledLabel) - .addComponent(getEnabledCheckBox()))); - - return fieldsPanel; - } - - @Override - protected String getConfirmButtonLabel() { - return CONFIRM_BUTTON_LABEL; - } - - @Override - protected void init() { - getDomainTextField().setText(""); - getRegexCheckBox().setSelected(false); - getEnabledCheckBox().setSelected(true); - domainAlwaysInScope = null; - } - - @Override - protected boolean validateFields() { - if (getRegexCheckBox().isSelected()) { - try { - org.zaproxy.zap.spider.DomainAlwaysInScopeMatcher.createPattern( - getDomainTextField().getText()); - } catch (IllegalArgumentException e) { - JOptionPane.showMessageDialog( - this, - TEXT_INVALID_REGEX_DIALOG, - TITLE_INVALID_REGEX_DIALOG, - JOptionPane.INFORMATION_MESSAGE); - getDomainTextField().requestFocusInWindow(); - return false; - } - } - - return true; - } - - @Override - protected void performAction() { - String value = getDomainTextField().getText(); - if (getRegexCheckBox().isSelected()) { - Pattern pattern = - org.zaproxy.zap.spider.DomainAlwaysInScopeMatcher.createPattern(value); - domainAlwaysInScope = new org.zaproxy.zap.spider.DomainAlwaysInScopeMatcher(pattern); - } else { - domainAlwaysInScope = new org.zaproxy.zap.spider.DomainAlwaysInScopeMatcher(value); - } - - domainAlwaysInScope.setEnabled(getEnabledCheckBox().isSelected()); - } - - @Override - protected void clearFields() { - getDomainTextField().setText(""); - getDomainTextField().discardAllEdits(); - } - - public org.zaproxy.zap.spider.DomainAlwaysInScopeMatcher getDomainAlwaysInScope() { - return domainAlwaysInScope; - } - - protected ZapTextField getDomainTextField() { - if (domainTextField == null) { - domainTextField = new ZapTextField(25); - domainTextField - .getDocument() - .addDocumentListener(getConfirmButtonValidatorDocListener()); - } - - return domainTextField; - } - - protected JCheckBox getRegexCheckBox() { - if (regexCheckBox == null) { - regexCheckBox = new JCheckBox(); - } - return regexCheckBox; - } - - protected JCheckBox getEnabledCheckBox() { - if (enabledCheckBox == null) { - enabledCheckBox = new JCheckBox(); - } - - return enabledCheckBox; - } - - public void clear() { - this.domainAlwaysInScope = null; - } - - private ConfirmButtonValidatorDocListener getConfirmButtonValidatorDocListener() { - if (confirmButtonValidatorDocListener == null) { - confirmButtonValidatorDocListener = new ConfirmButtonValidatorDocListener(); - } - return confirmButtonValidatorDocListener; - } - - private class ConfirmButtonValidatorDocListener implements DocumentListener { - - @Override - public void insertUpdate(DocumentEvent e) { - checkAndEnableConfirmButton(); - } - - @Override - public void removeUpdate(DocumentEvent e) { - checkAndEnableConfirmButton(); - } - - @Override - public void changedUpdate(DocumentEvent e) { - checkAndEnableConfirmButton(); - } - - private void checkAndEnableConfirmButton() { - boolean enabled = (getDomainTextField().getDocument().getLength() > 0); - setConfirmButtonEnabled(enabled); - } - } -} diff --git a/zap/src/main/java/org/zaproxy/zap/extension/spider/DialogModifyDomainAlwaysInScope.java b/zap/src/main/java/org/zaproxy/zap/extension/spider/DialogModifyDomainAlwaysInScope.java deleted file mode 100644 index 94700e17b5f..00000000000 --- a/zap/src/main/java/org/zaproxy/zap/extension/spider/DialogModifyDomainAlwaysInScope.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2014 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.extension.spider; - -import java.awt.Dialog; -import org.parosproxy.paros.Constant; - -/** - * @deprecated (2.12.0) See the spider add-on in zap-extensions instead. - */ -@Deprecated -class DialogModifyDomainAlwaysInScope extends DialogAddDomainAlwaysInScope { - - private static final long serialVersionUID = -4031122965844883255L; - - private static final String DIALOG_TITLE = - Constant.messages.getString("spider.options.domains.in.scope.modify.title"); - - private static final String CONFIRM_BUTTON_LABEL = - Constant.messages.getString("spider.options.domains.in.scope.modify.button.confirm"); - - protected DialogModifyDomainAlwaysInScope(Dialog owner) { - super(owner, DIALOG_TITLE); - } - - @Override - protected String getConfirmButtonLabel() { - return CONFIRM_BUTTON_LABEL; - } - - public void setDomainAlwaysInScope( - org.zaproxy.zap.spider.DomainAlwaysInScopeMatcher excludedDomain) { - this.domainAlwaysInScope = excludedDomain; - } - - @Override - protected void init() { - getDomainTextField().setText(domainAlwaysInScope.getValue()); - getDomainTextField().discardAllEdits(); - - getRegexCheckBox().setSelected(domainAlwaysInScope.isRegex()); - - getEnabledCheckBox().setSelected(domainAlwaysInScope.isEnabled()); - } -} diff --git a/zap/src/main/java/org/zaproxy/zap/extension/spider/DomainsAlwaysInScopeTableModel.java b/zap/src/main/java/org/zaproxy/zap/extension/spider/DomainsAlwaysInScopeTableModel.java deleted file mode 100644 index a5233f5904f..00000000000 --- a/zap/src/main/java/org/zaproxy/zap/extension/spider/DomainsAlwaysInScopeTableModel.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2014 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.extension.spider; - -import java.util.ArrayList; -import java.util.List; -import org.parosproxy.paros.Constant; -import org.zaproxy.zap.view.AbstractMultipleOptionsTableModel; - -@SuppressWarnings("serial") -/** - * @deprecated (2.12.0) See the spider add-on in zap-extensions instead. - */ -@Deprecated -class DomainsAlwaysInScopeTableModel - extends AbstractMultipleOptionsTableModel< - org.zaproxy.zap.spider.DomainAlwaysInScopeMatcher> { - - private static final long serialVersionUID = -5411351965957264957L; - - private static final String[] COLUMN_NAMES = { - Constant.messages.getString("spider.options.domains.in.scope.table.header.enabled"), - Constant.messages.getString("spider.options.domains.in.scope.table.header.regex"), - Constant.messages.getString("spider.options.domains.in.scope.table.header.value") - }; - - private static final int COLUMN_COUNT = COLUMN_NAMES.length; - - private List domainsInScope = - new ArrayList<>(5); - - public DomainsAlwaysInScopeTableModel() { - super(); - } - - @Override - public String getColumnName(int col) { - return COLUMN_NAMES[col]; - } - - @Override - public int getColumnCount() { - return COLUMN_COUNT; - } - - @Override - public int getRowCount() { - return domainsInScope.size(); - } - - @Override - public boolean isCellEditable(int rowIndex, int columnIndex) { - return (columnIndex == 0); - } - - @Override - public Object getValueAt(int rowIndex, int columnIndex) { - switch (columnIndex) { - case 0: - return getElement(rowIndex).isEnabled(); - case 1: - return getElement(rowIndex).isRegex(); - case 2: - return getElement(rowIndex).getValue(); - } - return null; - } - - @Override - public void setValueAt(Object aValue, int rowIndex, int columnIndex) { - if (columnIndex == 0 && aValue instanceof Boolean) { - domainsInScope.get(rowIndex).setEnabled((Boolean) aValue); - fireTableCellUpdated(rowIndex, columnIndex); - } - } - - @Override - public Class getColumnClass(int c) { - if (c == 0 || c == 1) { - return Boolean.class; - } - return String.class; - } - - public List getDomainsAlwaysInScope() { - return domainsInScope; - } - - public void setDomainsAlwaysInScope( - List domainsInScope) { - this.domainsInScope = new ArrayList<>(domainsInScope.size()); - - for (org.zaproxy.zap.spider.DomainAlwaysInScopeMatcher excludedDomain : domainsInScope) { - this.domainsInScope.add( - new org.zaproxy.zap.spider.DomainAlwaysInScopeMatcher(excludedDomain)); - } - - fireTableDataChanged(); - } - - @Override - public List getElements() { - return domainsInScope; - } -} diff --git a/zap/src/main/java/org/zaproxy/zap/extension/spider/ExtensionSpider.java b/zap/src/main/java/org/zaproxy/zap/extension/spider/ExtensionSpider.java deleted file mode 100644 index 93d64cb6daf..00000000000 --- a/zap/src/main/java/org/zaproxy/zap/extension/spider/ExtensionSpider.java +++ /dev/null @@ -1,898 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2010 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.extension.spider; - -import java.awt.Dimension; -import java.awt.EventQueue; -import java.awt.event.KeyEvent; -import java.text.MessageFormat; -import java.util.ArrayList; -import java.util.Collections; -import java.util.LinkedList; -import java.util.List; -import javax.swing.Icon; -import javax.swing.ImageIcon; -import org.apache.commons.httpclient.URI; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.parosproxy.paros.Constant; -import org.parosproxy.paros.control.Control; -import org.parosproxy.paros.control.Control.Mode; -import org.parosproxy.paros.extension.ExtensionAdaptor; -import org.parosproxy.paros.extension.ExtensionHook; -import org.parosproxy.paros.extension.SessionChangedListener; -import org.parosproxy.paros.model.Session; -import org.parosproxy.paros.model.SiteNode; -import org.zaproxy.zap.extension.help.ExtensionHelp; -import org.zaproxy.zap.model.Context; -import org.zaproxy.zap.model.ScanController; -import org.zaproxy.zap.model.StructuralNode; -import org.zaproxy.zap.model.StructuralSiteNode; -import org.zaproxy.zap.model.Target; -import org.zaproxy.zap.users.User; -import org.zaproxy.zap.view.ZapMenuItem; - -/** - * The ExtensionSpider is the Extension that controls the Spider. - * - * @deprecated (2.12.0) See the spider add-on in zap-extensions instead. - */ -@Deprecated -@SuppressWarnings("removal") -public class ExtensionSpider extends ExtensionAdaptor - implements SessionChangedListener, ScanController { - - private org.zaproxy.zap.model.ValueGenerator generator = - new org.zaproxy.zap.model.DefaultValueGenerator(); - - public static final int EXTENSION_ORDER = 30; - - /** The Constant logger. */ - private static final Logger log = LogManager.getLogger(ExtensionSpider.class); - - /** The Constant defining the NAME of the extension. */ - public static final String NAME = "ExtensionSpider"; - - /** The spider panel. */ - private SpiderPanel spiderPanel = null; - - SpiderDialog spiderDialog = null; - - private PopupMenuItemSpiderDialog popupMenuItemSpiderDialog; - - private PopupMenuItemSpiderDialogWithContext popupMenuItemSpiderDialogWithContext; - - /** The options spider panel. */ - private OptionsSpiderPanel optionsSpiderPanel = null; - - /** The params for the spider. */ - private org.zaproxy.zap.spider.SpiderParam params = null; - - private List customParsers; - private List customFetchFilters; - private List customParseFilters; - - private SpiderAPI spiderApi; - - private SpiderScanController scanController = null; - - private Icon icon; - - private boolean panelSwitch = true; - - /** - * The list of excluded patterns of sites. Patterns are added here with the ExcludeFromSpider - * Popup Menu. - */ - private List excludeList = Collections.emptyList(); - - private ZapMenuItem menuItemCustomScan = null; - - /** Instantiates a new spider extension. */ - public ExtensionSpider() { - super(NAME); - initialize(); - } - - /** This method initializes this extension. */ - private void initialize() { - this.setOrder(EXTENSION_ORDER); - this.customParsers = new LinkedList<>(); - this.customFetchFilters = new LinkedList<>(); - this.customParseFilters = new LinkedList<>(); - this.scanController = new SpiderScanController(this); - } - - public void setValueGenerator(org.zaproxy.zap.model.ValueGenerator generator) { - if (generator == null) { - throw new IllegalArgumentException("Parameter generator must not be null."); - } - this.generator = generator; - } - - public org.zaproxy.zap.model.ValueGenerator getValueGenerator() { - return generator; - } - - @Override - public String getUIName() { - return Constant.messages.getString("spider.name"); - } - - @Override - public void hook(ExtensionHook extensionHook) { - super.hook(extensionHook); - // Register for listeners - extensionHook.addSessionListener(this); - - // Initialize views - if (getView() != null) { - extensionHook.getHookMenu().addToolsMenuItem(getMenuItemCustomScan()); - extensionHook.getHookView().addStatusPanel(getSpiderPanel()); - extensionHook.getHookView().addOptionPanel(getOptionsSpiderPanel()); - extensionHook.getHookMenu().addPopupMenuItem(getPopupMenuItemSpiderDialog()); - extensionHook.getHookMenu().addPopupMenuItem(getPopupMenuItemSpiderDialogWithContext()); - ExtensionHelp.enableHelpKey(getSpiderPanel(), "ui.tabs.spider"); - } - - // Register the params - extensionHook.addOptionsParamSet(getSpiderParam()); - - // Register as an API implementor - spiderApi = new SpiderAPI(this); - spiderApi.addApiOptions(getSpiderParam()); - extensionHook.addApiImplementor(spiderApi); - } - - private PopupMenuItemSpiderDialog getPopupMenuItemSpiderDialog() { - if (popupMenuItemSpiderDialog == null) { - popupMenuItemSpiderDialog = new PopupMenuItemSpiderDialog(this); - } - return popupMenuItemSpiderDialog; - } - - private PopupMenuItemSpiderDialogWithContext getPopupMenuItemSpiderDialogWithContext() { - if (popupMenuItemSpiderDialogWithContext == null) { - popupMenuItemSpiderDialogWithContext = new PopupMenuItemSpiderDialogWithContext(this); - } - return popupMenuItemSpiderDialogWithContext; - } - - @Override - public List getActiveActions() { - List activeSpiders = scanController.getActiveScans(); - if (activeSpiders.isEmpty()) { - return null; - } - - String spiderActionPrefix = Constant.messages.getString("spider.activeActionPrefix"); - List activeActions = new ArrayList<>(activeSpiders.size()); - for (SpiderScan activeSpider : activeSpiders) { - activeActions.add( - MessageFormat.format(spiderActionPrefix, activeSpider.getDisplayName())); - } - return activeActions; - } - - /** - * Gets the spider parameters (options). - * - * @return the spider parameters - */ - protected org.zaproxy.zap.spider.SpiderParam getSpiderParam() { - if (params == null) { - params = new org.zaproxy.zap.spider.SpiderParam(); - } - return params; - } - - /** - * Gets the spider panel. - * - * @return the spider panel - */ - protected SpiderPanel getSpiderPanel() { - if (spiderPanel == null) { - spiderPanel = new SpiderPanel(this, getSpiderParam()); - } - return spiderPanel; - } - - @Override - public void sessionAboutToChange(Session session) { - // Shut all of the scans down and remove them - this.scanController.reset(); - if (hasView()) { - this.getSpiderPanel().reset(); - if (spiderDialog != null) { - spiderDialog.reset(); - } - } - } - - @Override - public void sessionChanged(final Session session) { - if (EventQueue.isDispatchThread()) { - sessionChangedEventHandler(session); - } else { - try { - EventQueue.invokeAndWait( - new Runnable() { - @Override - public void run() { - sessionChangedEventHandler(session); - } - }); - } catch (Exception e) { - log.error(e.getMessage(), e); - } - } - } - - /** - * Session changed event handler. - * - * @param session the session - */ - private void sessionChangedEventHandler(Session session) { - // Clear all scans - if (hasView()) { - this.getSpiderPanel().reset(); - } - if (session == null) { - // Closedown - return; - } - } - - /** - * Gets the options spider panel. - * - * @return the options spider panel - */ - private OptionsSpiderPanel getOptionsSpiderPanel() { - if (optionsSpiderPanel == null) { - optionsSpiderPanel = new OptionsSpiderPanel(); - } - return optionsSpiderPanel; - } - - /** - * Sets the exclude list. - * - * @param ignoredRegexs the new exclude list - */ - public void setExcludeList(List ignoredRegexs) { - if (ignoredRegexs == null || ignoredRegexs.isEmpty()) { - excludeList = Collections.emptyList(); - return; - } - - this.excludeList = ignoredRegexs; - } - - /** - * Gets the exclude list. - * - * @return the exclude list - */ - public List getExcludeList() { - return excludeList; - } - - @Override - public String getAuthor() { - return Constant.ZAP_TEAM; - } - - @Override - public String getDescription() { - return Constant.messages.getString("spider.desc"); - } - - @Override - public void sessionScopeChanged(Session session) { - if (hasView()) { - this.getSpiderPanel().sessionScopeChanged(session); - } - } - - @Override - public void sessionModeChanged(Mode mode) { - if (Mode.safe.equals(mode)) { - this.scanController.stopAllScans(); - } - - if (hasView()) { - this.getSpiderPanel().sessionModeChanged(mode); - getMenuItemCustomScan().setEnabled(!Mode.safe.equals(mode)); - } - } - - /** - * Start scan node. - * - * @param node the node - */ - public void startScanNode(SiteNode node) { - Target target = new Target(node); - target.setRecurse(true); - this.startScan(target, null, null); - } - - /** - * Start the scan of an URL (Node) from the POV of a User. - * - * @param node the node - */ - public void startScanNode(SiteNode node, User user) { - Target target = new Target(node); - target.setRecurse(true); - this.startScan(target, user, null); - } - - /** Start scan all in scope. */ - public void startScanAllInScope() { - Target target = new Target(true); - target.setRecurse(true); - this.startScan(target, null, null); - } - - /** - * Start scan. - * - * @param startNode the start node - */ - public void startScan(SiteNode startNode) { - Target target = new Target(startNode); - target.setRecurse(true); - this.startScan(target, null, null); - } - - /** Start scan all in context, from the POV of an User. */ - public void startScanAllInContext(Context context, User user) { - Target target = new Target(context); - target.setRecurse(true); - this.startScan(target, user, null); - } - - @Override - public void destroy() { - // Shut all of the scans down - this.stopAllScans(); - if (hasView()) { - this.getSpiderPanel().reset(); - } - } - - /** - * Gets the custom parsers loaded. - * - * @return the custom parsers - */ - public List getCustomParsers() { - return customParsers; - } - - /** - * Gets the custom fetch filters loaded. - * - * @return the custom fetch filters - */ - public List getCustomFetchFilters() { - return customFetchFilters; - } - - /** - * Gets the custom parse filters loaded. - * - * @return the custom parse filters - */ - public List getCustomParseFilters() { - return customParseFilters; - } - - /** - * Adds a new custom Spider Parser. The parser is added at the beginning of the parsers list so - * it will be processed before other already loaded parsers and before the default parsers. - * - *

This method should be used to customize the Spider from any other extension of ZAP. The - * parsers added will be loaded whenever starting any scan. - * - * @param parser the parser - * @throws IllegalArgumentException if the given parameter is {@code null}. - * @see #removeCustomParser(org.zaproxy.zap.spider.parser.SpiderParser) - */ - public void addCustomParser(org.zaproxy.zap.spider.parser.SpiderParser parser) { - validateParameterNonNull(parser, "parser"); - this.customParsers.add(parser); - } - - private static void validateParameterNonNull(Object object, String name) { - if (object == null) { - throw new IllegalArgumentException("Parameter " + name + " must not be null."); - } - } - - /** - * Removes the given spider parser. - * - *

Nothing happens if the given parser was not previously added. - * - * @param parser the parser - * @throws IllegalArgumentException if the given parameter is {@code null}. - * @since 2.6.0 - * @see #addCustomParser(org.zaproxy.zap.spider.parser.SpiderParser) - */ - public void removeCustomParser(org.zaproxy.zap.spider.parser.SpiderParser parser) { - validateParameterNonNull(parser, "parser"); - this.customParsers.remove(parser); - } - - /** - * Adds a custom fetch filter that would be used during the spidering. - * - *

This method should be used to customize the Spider from any other extension of ZAP. The - * filters added will be loaded whenever starting any scan. - * - * @param filter the filter - * @throws IllegalArgumentException if the given parameter is {@code null}. - * @see #removeCustomFetchFilter(org.zaproxy.zap.spider.filters.FetchFilter) - */ - public void addCustomFetchFilter(org.zaproxy.zap.spider.filters.FetchFilter filter) { - validateParameterNonNull(filter, "filter"); - this.customFetchFilters.add(filter); - } - - /** - * Removes the given fetch filter. - * - *

Nothing happens if the given filter was not previously added. - * - * @param filter the filter - * @throws IllegalArgumentException if the given parameter is {@code null}. - * @since 2.6.0 - * @see #addCustomFetchFilter(org.zaproxy.zap.spider.filters.FetchFilter) - */ - public void removeCustomFetchFilter(org.zaproxy.zap.spider.filters.FetchFilter filter) { - validateParameterNonNull(filter, "filter"); - this.customFetchFilters.remove(filter); - } - - /** - * Adds a custom parse filter that would be used during the spidering. - * - *

This method should be used to customize the Spider from any other extension of ZAP. The - * filters added will be loaded whenever starting any scan. - * - * @param filter the filter - * @throws IllegalArgumentException if the given parameter is {@code null}. - * @see #removeCustomParseFilter(org.zaproxy.zap.spider.filters.ParseFilter) - */ - public void addCustomParseFilter(org.zaproxy.zap.spider.filters.ParseFilter filter) { - validateParameterNonNull(filter, "filter"); - this.customParseFilters.add(filter); - } - - /** - * Removes the given parse filter. - * - *

Nothing happens if the given filter was not previously added. - * - * @param filter the filter - * @throws IllegalArgumentException if the given parameter is {@code null}. - * @since 2.6.0 - * @see #addCustomParseFilter(org.zaproxy.zap.spider.filters.ParseFilter) - */ - public void removeCustomParseFilter(org.zaproxy.zap.spider.filters.ParseFilter filter) { - validateParameterNonNull(filter, "filter"); - this.customParseFilters.remove(filter); - } - - /** - * Starts a new spider scan using the given target and, optionally, spidering from the - * perspective of a user and with custom configurations. - * - *

The spider scan will use the most appropriate display name created from the given target, - * user and custom configurations. - * - * @param target the target that will be spidered - * @param user the user that will be used to spider, might be {@code null} - * @param customConfigurations other custom configurations for the spider, might be {@code null} - * @return the ID of the spider scan - * @since 2.5.0 - * @see #startScan(String, Target, User, Object[]) - * @throws IllegalStateException if the target or custom configurations are not allowed in the - * current {@link org.parosproxy.paros.control.Control.Mode mode}. - */ - public int startScan(Target target, User user, Object[] customConfigurations) { - return startScan( - createDisplayName(target, customConfigurations), - target, - user, - customConfigurations); - } - - /** - * Creates the display name for the given target and, optionally, the given custom - * configurations. - * - * @param target the target that will be spidered - * @param customConfigurations other custom configurations for the spider, might be {@code null} - * @return a {@code String} containing the display name, never {@code null} - */ - private String createDisplayName(Target target, Object[] customConfigurations) { - org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter subtreeFecthFilter = - getUriPrefixFecthFilter(customConfigurations); - if (subtreeFecthFilter != null) { - return abbreviateDisplayName(subtreeFecthFilter.getNormalisedPrefix()); - } - - if (target.getContext() != null) { - return Constant.messages.getString("context.prefixName", target.getContext().getName()); - } else if (target.isInScopeOnly()) { - return Constant.messages.getString("target.allInScope"); - } else if (target.getStartNode() == null) { - if (customConfigurations != null) { - for (Object customConfiguration : customConfigurations) { - if (customConfiguration instanceof URI) { - return abbreviateDisplayName(((URI) customConfiguration).toString()); - } - } - } - return Constant.messages.getString("target.empty"); - } - return abbreviateDisplayName(target.getStartNode().getHierarchicNodeName(false)); - } - - /** - * Gets the {@code HttpPrefixFetchFilter} from the given {@code customConfigurations}. - * - * @param customConfigurations the custom configurations of the spider - * @return the {@code HttpPrefixFetchFilter} found, {@code null} otherwise. - */ - private org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter getUriPrefixFecthFilter( - Object[] customConfigurations) { - if (customConfigurations != null) { - for (Object customConfiguration : customConfigurations) { - if (customConfiguration - instanceof org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter) { - return (org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter) - customConfiguration; - } - } - } - return null; - } - - /** - * Abbreviates (the middle of) the given display name if greater than 30 characters. - * - * @param displayName the display name that might be abbreviated - * @return the, possibly, abbreviated display name - */ - private static String abbreviateDisplayName(String displayName) { - return StringUtils.abbreviateMiddle(displayName, "..", 30); - } - - /** - * Starts a new spider scan, with the given display name, using the given target and, - * optionally, spidering from the perspective of a user and with custom configurations. - * - *

Note: The preferred method to start the scan is with {@link - * #startScan(Target, User, Object[])}, unless a custom display name is really needed. - * - * @param target the target that will be spidered - * @param user the user that will be used to spider, might be {@code null} - * @param customConfigurations other custom configurations for the spider, might be {@code null} - * @return the ID of the spider scan - * @throws IllegalStateException if the target or custom configurations are not allowed in the - * current {@link org.parosproxy.paros.control.Control.Mode mode}. - */ - @SuppressWarnings({"fallthrough"}) - @Override - public int startScan( - String displayName, Target target, User user, Object[] customConfigurations) { - switch (Control.getSingleton().getMode()) { - case safe: - throw new IllegalStateException("Scans are not allowed in Safe mode"); - case protect: - String uri = getTargetUriOutOfScope(target, customConfigurations); - if (uri != null) { - throw new IllegalStateException( - "Scans are not allowed on targets not in scope when in Protected mode: " - + uri); - } - // $FALL-THROUGH$ - case standard: - case attack: - // No problem - break; - } - - int id = this.scanController.startScan(displayName, target, user, customConfigurations); - if (hasView()) { - addScanToUi(this.scanController.getScan(id)); - } - return id; - } - - private void addScanToUi(final SpiderScan scan) { - if (!EventQueue.isDispatchThread()) { - EventQueue.invokeLater( - new Runnable() { - - @Override - public void run() { - addScanToUi(scan); - } - }); - return; - } - - this.getSpiderPanel().scannerStarted(scan); - scan.setListener(getSpiderPanel()); // So the UI gets updated - this.getSpiderPanel().switchView(scan); - if (isPanelSwitch()) { - this.getSpiderPanel().setTabFocus(); - } - } - - /** - * Returns true if the GUI will switch to the Spider panel when a scan is started. - * - * @since 2.11.0 - */ - public boolean isPanelSwitch() { - return panelSwitch; - } - - /** - * Sets if the GUI will switch to the Spider panel when a scan is started. Code should only set - * this to false just before starting a scan and reset it to true as soon as the scan has - * started. - * - * @since 2.11.0 - */ - public void setPanelSwitch(boolean panelSwitch) { - this.panelSwitch = panelSwitch; - } - - /** - * Returns the first URI that is out of scope in the given {@code target}. - * - * @param target the target that will be checked - * @return a {@code String} with the first URI out of scope, {@code null} if none found - * @since 2.5.0 - * @see Session#isInScope(String) - */ - protected String getTargetUriOutOfScope(Target target) { - return getTargetUriOutOfScope(target, null); - } - - /** - * Returns the first URI that is out of scope in the given {@code target} or {@code - * contextSpecificObjects}. - * - * @param target the target that will be checked - * @param contextSpecificObjects other {@code Objects} used to enhance the target - * @return a {@code String} with the first URI out of scope, {@code null} if none found - * @since 2.5.0 - * @see Session#isInScope(String) - */ - protected String getTargetUriOutOfScope(Target target, Object[] contextSpecificObjects) { - List nodes = target.getStartNodes(); - if (nodes != null) { - for (StructuralNode node : nodes) { - if (node == null) { - continue; - } - if (node instanceof StructuralSiteNode) { - SiteNode siteNode = ((StructuralSiteNode) node).getSiteNode(); - if (!siteNode.isIncludedInScope()) { - return node.getURI().toString(); - } - } else { - String uri = node.getURI().toString(); - if (!isTargetUriInScope(uri)) { - return uri; - } - } - } - } - if (contextSpecificObjects != null) { - for (Object obj : contextSpecificObjects) { - if (obj instanceof URI) { - String uri = ((URI) obj).toString(); - if (!isTargetUriInScope(uri)) { - return uri; - } - } - } - } - return null; - } - - /** - * Tells whether or not the given {@code uri} is in scope. - * - * @param uri the uri that will be checked - * @return {@code true} if the {@code uri} is in scope, {@code false} otherwise - * @since 2.5.0 - * @see Session#isInScope(String) - */ - protected boolean isTargetUriInScope(String uri) { - if (uri == null) { - return false; - } - return getModel().getSession().isInScope(uri); - } - - @Override - public List getAllScans() { - return this.scanController.getAllScans(); - } - - @Override - public List getActiveScans() { - return this.scanController.getActiveScans(); - } - - @Override - public SpiderScan getScan(int id) { - return this.scanController.getScan(id); - } - - @Override - public void stopScan(int id) { - this.scanController.stopScan(id); - } - - @Override - public void stopAllScans() { - this.scanController.stopAllScans(); - } - - @Override - public void pauseScan(int id) { - this.scanController.pauseScan(id); - if (hasView()) { - // Update the UI in case this was initiated from the API - this.getSpiderPanel().updateScannerUI(); - } - } - - @Override - public void pauseAllScans() { - this.scanController.pauseAllScans(); - if (hasView()) { - // Update the UI in case this was initiated from the API - this.getSpiderPanel().updateScannerUI(); - } - } - - @Override - public void resumeScan(int id) { - this.scanController.resumeScan(id); - if (hasView()) { - // Update the UI in case this was initiated from the API - this.getSpiderPanel().updateScannerUI(); - } - } - - @Override - public void resumeAllScans() { - this.scanController.resumeAllScans(); - if (hasView()) { - // Update the UI in case this was initiated from the API - this.getSpiderPanel().updateScannerUI(); - } - } - - @Override - public SpiderScan removeScan(int id) { - return this.scanController.removeScan(id); - } - - @Override - public int removeAllScans() { - return this.scanController.removeAllScans(); - } - - @Override - public int removeFinishedScans() { - return this.scanController.removeFinishedScans(); - } - - @Override - public SpiderScan getLastScan() { - return this.scanController.getLastScan(); - } - - private ZapMenuItem getMenuItemCustomScan() { - if (menuItemCustomScan == null) { - menuItemCustomScan = - new ZapMenuItem( - "menu.tools.spider", - getView() - .getMenuShortcutKeyStroke( - KeyEvent.VK_S, KeyEvent.ALT_DOWN_MASK, false)); - menuItemCustomScan.setEnabled(Control.getSingleton().getMode() != Mode.safe); - - menuItemCustomScan.addActionListener(e -> showSpiderDialog((Target) null)); - } - - return menuItemCustomScan; - } - - public void showSpiderDialog(SiteNode node) { - showSpiderDialog(node != null ? new Target(node) : null); - } - - /** - * Shows the spider dialogue with the given target, if not already visible. - * - * @param target the target, might be {@code null}. - * @since 2.8.0. - */ - public void showSpiderDialog(Target target) { - if (spiderDialog == null) { - spiderDialog = - new SpiderDialog(this, getView().getMainFrame(), new Dimension(700, 430)); - } - if (spiderDialog.isVisible()) { - // Its behind you! Actually not needed no the window is alwaysOnTop, but keeping in case - // we change that ;) - spiderDialog.toFront(); - return; - } - if (target != null) { - spiderDialog.init(target); - } else { - // Keep the previous target - spiderDialog.init(null); - } - spiderDialog.setVisible(true); - } - - @Override - public boolean supportsLowMemory() { - return true; - } - - /** No database tables used, so all supported */ - @Override - public boolean supportsDb(String type) { - return true; - } - - /** - * Gets the icon for spider related functionality. - * - * @return the icon - */ - public Icon getIcon() { - if (icon == null) { - icon = new ImageIcon(ExtensionSpider.class.getResource("/resource/icon/16/spider.png")); - } - return icon; - } -} diff --git a/zap/src/main/java/org/zaproxy/zap/extension/spider/OptionsSpiderPanel.java b/zap/src/main/java/org/zaproxy/zap/extension/spider/OptionsSpiderPanel.java deleted file mode 100644 index f3b461106e2..00000000000 --- a/zap/src/main/java/org/zaproxy/zap/extension/spider/OptionsSpiderPanel.java +++ /dev/null @@ -1,649 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2012 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.extension.spider; - -import java.awt.BorderLayout; -import java.awt.CardLayout; -import java.awt.Component; -import java.awt.GridBagConstraints; -import java.awt.GridBagLayout; -import java.awt.Insets; -import javax.swing.BorderFactory; -import javax.swing.JCheckBox; -import javax.swing.JComboBox; -import javax.swing.JLabel; -import javax.swing.JList; -import javax.swing.JOptionPane; -import javax.swing.JPanel; -import javax.swing.JScrollPane; -import javax.swing.JSlider; -import javax.swing.ScrollPaneConstants; -import javax.swing.SortOrder; -import javax.swing.border.Border; -import javax.swing.border.EmptyBorder; -import javax.swing.event.ChangeEvent; -import javax.swing.event.ChangeListener; -import javax.swing.plaf.basic.BasicComboBoxRenderer; -import org.parosproxy.paros.Constant; -import org.parosproxy.paros.model.Model; -import org.parosproxy.paros.model.OptionsParam; -import org.parosproxy.paros.view.AbstractParamPanel; -import org.parosproxy.paros.view.View; -import org.zaproxy.zap.utils.ZapNumberSpinner; -import org.zaproxy.zap.utils.ZapTextArea; -import org.zaproxy.zap.view.AbstractMultipleOptionsTablePanel; -import org.zaproxy.zap.view.LayoutHelper; -import org.zaproxy.zap.view.PositiveValuesSlider; - -/** - * The Class OptionsSpiderPanel defines the Options Panel showed when configuring settings related - * to the spider. - * - * @deprecated (2.12.0) See the spider add-on in zap-extensions instead. - */ -@Deprecated -public class OptionsSpiderPanel extends AbstractParamPanel { - - /** The Constant serialVersionUID. */ - private static final long serialVersionUID = -5623691753271231473L; - - /** The full panel for the spider options. */ - private JPanel panelSpider = null; - - // The controls for the options: - private JSlider sliderMaxDepth = null; - private JSlider sliderThreads = null; - private ZapNumberSpinner durationNumberSpinner = null; - private ZapNumberSpinner maxChildrenNumberSpinner; - private ZapNumberSpinner maxParseSizeBytesNumberSpinner; - private JCheckBox chkPostForm = null; - private JCheckBox chkProcessForm = null; - private JCheckBox parseComments = null; - private JCheckBox parseRobotsTxt = null; - private JCheckBox parseSitemapXml = null; - private JCheckBox parseSVNEntries = null; - private JCheckBox parseGit = null; - private JCheckBox handleODataSpecificParameters = null; - private JCheckBox chkSendRefererHeader; - private JCheckBox chkAcceptCookies; - private DomainsAlwaysInScopeMultipleOptionsPanel domainsAlwaysInScopePanel; - private DomainsAlwaysInScopeTableModel domainsAlwaysInScopeTableModel; - private ZapTextArea irrelevantUrlParameters; - - private JComboBox handleParameters = - null; - - /** Instantiates a new options spider panel. */ - public OptionsSpiderPanel() { - super(); - initialize(); - } - - /** This method initializes this options Panel. */ - private void initialize() { - this.setLayout(new CardLayout()); - this.setName(Constant.messages.getString("spider.options.title")); - if (Model.getSingleton().getOptionsParam().getViewParam().getWmUiHandlingOption() == 0) { - this.setSize(314, 245); - } - this.add(getPanelSpider(), getPanelSpider().getName()); - } - - /** - * This method initializes the main panel containing all option controls. - * - * @return the panel for the spider options. - */ - private JPanel getPanelSpider() { - if (panelSpider == null) { - - // Initialize the panel - panelSpider = new JPanel(new BorderLayout()); - if (Model.getSingleton().getOptionsParam().getViewParam().getWmUiHandlingOption() - == 0) { - panelSpider.setSize(114, 150); - } - panelSpider.setName(""); - - // Prepare the necessary labels - JLabel domainsLabel = new JLabel(); - JLabel noThreadsLabel = new JLabel(); - JLabel maxDuration = new JLabel(); - JLabel maxDepthLabel = new JLabel(); - JLabel handleParametersLabel = new JLabel(); - - maxDepthLabel.setText(Constant.messages.getString("spider.options.label.depth")); - noThreadsLabel.setText(Constant.messages.getString("spider.options.label.threads")); - maxDuration.setText(Constant.messages.getString("spider.options.label.duration")); - domainsLabel.setText(Constant.messages.getString("spider.options.label.domains")); - handleParametersLabel.setText( - Constant.messages.getString("spider.options.label.handleparameters")); - - JPanel innerPanel = new JPanel(new GridBagLayout()); - - GridBagConstraints gbc = new GridBagConstraints(); - gbc.gridx = 0; - gbc.weightx = 1.0D; - gbc.fill = GridBagConstraints.HORIZONTAL; - gbc.anchor = GridBagConstraints.NORTHWEST; - Insets insets = new Insets(2, 2, 2, 2); - gbc.insets = insets; - - // Add the components on the panel - innerPanel.add(maxDepthLabel, gbc); - innerPanel.add(getSliderMaxDepth(), gbc); - innerPanel.add(noThreadsLabel, gbc); - innerPanel.add(getSliderThreads(), gbc); - - JPanel inlineOptionsPanel = new JPanel(new GridBagLayout()); - inlineOptionsPanel.add(maxDuration, LayoutHelper.getGBC(0, 0, 1, 1.0D)); - inlineOptionsPanel.add(getDurationNumberSpinner(), LayoutHelper.getGBC(1, 0, 1, 1.0D)); - - inlineOptionsPanel.add( - new JLabel(Constant.messages.getString("spider.options.label.maxChildren")), - LayoutHelper.getGBC(0, 1, 1, 1.0D)); - inlineOptionsPanel.add( - getMaxChildrenNumberSpinner(), LayoutHelper.getGBC(1, 1, 1, 1.0D)); - - inlineOptionsPanel.add( - new JLabel( - Constant.messages.getString("spider.options.label.maxParseSizeBytes")), - LayoutHelper.getGBC(0, 2, 1, 1.0D)); - inlineOptionsPanel.add( - getMaxParseSizeBytesNumberSpinner(), LayoutHelper.getGBC(1, 2, 1, 1.0D)); - - innerPanel.add(inlineOptionsPanel, gbc); - - innerPanel.add(domainsLabel, gbc); - gbc.fill = GridBagConstraints.BOTH; - gbc.weighty = 1.0D; - innerPanel.add(getDomainsAlwaysInScopePanel(), gbc); - gbc.fill = GridBagConstraints.HORIZONTAL; - gbc.weighty = 0; - innerPanel.add(getChkSendRefererHeader(), gbc); - innerPanel.add(getChkAcceptCookies(), gbc); - innerPanel.add(handleParametersLabel, gbc); - innerPanel.add(getComboHandleParameters(), gbc); - innerPanel.add(getChkProcessForm(), gbc); - insets.left = 15; - innerPanel.add(getChkPostForm(), gbc); - insets.left = 2; - innerPanel.add(getChkParseComments(), gbc); - innerPanel.add(getChkParseRobotsTxt(), gbc); - innerPanel.add(getChkParseSitemapXml(), gbc); - innerPanel.add(getChkParseSVNEntries(), gbc); - innerPanel.add(getChkParseGit(), gbc); - innerPanel.add(getHandleODataSpecificParameters(), gbc); - - ZapTextArea irrelevantUrlParameters = getIrrelevantUrlParameters(); - JLabel label = - new JLabel( - Constant.messages.getString( - "spider.options.label.irrelevantUrlParameters")); - label.setLabelFor(irrelevantUrlParameters); - JScrollPane irrelevantUrlParametersScrollPane = new JScrollPane(); - irrelevantUrlParametersScrollPane.setVerticalScrollBarPolicy( - ScrollPaneConstants.VERTICAL_SCROLLBAR_AS_NEEDED); - irrelevantUrlParametersScrollPane.setViewportView(irrelevantUrlParameters); - - innerPanel.add(label, gbc); - innerPanel.add(irrelevantUrlParametersScrollPane, gbc); - - JScrollPane scrollPane = new JScrollPane(innerPanel); - scrollPane.setBorder(BorderFactory.createEmptyBorder()); - - panelSpider.add(scrollPane, BorderLayout.CENTER); - } - return panelSpider; - } - - @Override - public void initParam(Object obj) { - OptionsParam options = (OptionsParam) obj; - - org.zaproxy.zap.spider.SpiderParam param = - options.getParamSet(org.zaproxy.zap.spider.SpiderParam.class); - getSliderMaxDepth().setValue(param.getMaxDepth()); - getSliderThreads().setValue(param.getThreadCount()); - getDurationNumberSpinner().setValue(param.getMaxDuration()); - getMaxChildrenNumberSpinner().setValue(param.getMaxChildren()); - getMaxParseSizeBytesNumberSpinner().setValue(param.getMaxParseSizeBytes()); - getDomainsAlwaysInScopeTableModel() - .setDomainsAlwaysInScope(param.getDomainsAlwaysInScope()); - getDomainsAlwaysInScopePanel() - .setRemoveWithoutConfirmation(param.isConfirmRemoveDomainAlwaysInScope()); - getChkProcessForm().setSelected(param.isProcessForm()); - getChkSendRefererHeader().setSelected(param.isSendRefererHeader()); - getChkAcceptCookies().setSelected(param.isAcceptCookies()); - getChkPostForm().setSelected(param.isPostForm()); - getChkParseComments().setSelected(param.isParseComments()); - getChkParseRobotsTxt().setSelected(param.isParseRobotsTxt()); - getChkParseSitemapXml().setSelected(param.isParseSitemapXml()); - getChkParseSVNEntries().setSelected(param.isParseSVNEntries()); - getChkParseGit().setSelected(param.isParseGit()); - getComboHandleParameters().setSelectedItem(param.getHandleParameters()); - getHandleODataSpecificParameters().setSelected(param.isHandleODataParametersVisited()); - getIrrelevantUrlParameters().setText(param.getIrrelevantUrlParametersAsString()); - getIrrelevantUrlParameters().discardAllEdits(); - } - - @Override - public void saveParam(Object obj) throws Exception { - OptionsParam options = (OptionsParam) obj; - org.zaproxy.zap.spider.SpiderParam param = - options.getParamSet(org.zaproxy.zap.spider.SpiderParam.class); - param.setMaxDepth(getSliderMaxDepth().getValue()); - param.setThreadCount(getSliderThreads().getValue()); - param.setMaxDuration(getDurationNumberSpinner().getValue()); - param.setMaxChildren(getMaxChildrenNumberSpinner().getValue()); - param.setMaxParseSizeBytes(getMaxParseSizeBytesNumberSpinner().getValue()); - param.setDomainsAlwaysInScope( - getDomainsAlwaysInScopeTableModel().getDomainsAlwaysInScope()); - param.setConfirmRemoveDomainAlwaysInScope( - getDomainsAlwaysInScopePanel().isRemoveWithoutConfirmation()); - param.setSendRefererHeader(getChkSendRefererHeader().isSelected()); - param.setAcceptCookies(getChkAcceptCookies().isSelected()); - param.setProcessForm(getChkProcessForm().isSelected()); - param.setPostForm(getChkPostForm().isSelected()); - param.setParseComments(getChkParseComments().isSelected()); - param.setParseRobotsTxt(getChkParseRobotsTxt().isSelected()); - param.setParseSitemapXml(getChkParseSitemapXml().isSelected()); - param.setParseSVNEntries(getChkParseSVNEntries().isSelected()); - param.setParseGit(getChkParseGit().isSelected()); - param.setHandleParameters( - (org.zaproxy.zap.spider.SpiderParam.HandleParametersOption) - getComboHandleParameters().getSelectedItem()); - param.setHandleODataParametersVisited(getHandleODataSpecificParameters().isSelected()); - param.setIrrelevantUrlParameters(getIrrelevantUrlParameters().getText()); - } - - /** - * This method initializes the slider for MaxDepth. - * - * @return the slider for max depth - */ - private JSlider getSliderMaxDepth() { - if (sliderMaxDepth == null) { - sliderMaxDepth = new JSlider(); - sliderMaxDepth.setMaximum(19); - sliderMaxDepth.setMinimum(0); - sliderMaxDepth.setMinorTickSpacing(1); - sliderMaxDepth.setPaintTicks(true); - sliderMaxDepth.setPaintLabels(true); - sliderMaxDepth.setName(""); - sliderMaxDepth.setMajorTickSpacing(1); - sliderMaxDepth.setSnapToTicks(true); - sliderMaxDepth.setPaintTrack(true); - } - return sliderMaxDepth; - } - - /** - * This method initializes the slider for maximum number of threads used. - * - * @return javax.swing.JSlider - */ - private JSlider getSliderThreads() { - if (sliderThreads == null) { - sliderThreads = new PositiveValuesSlider(Constant.MAX_THREADS_PER_SCAN); - } - return sliderThreads; - } - - private ZapNumberSpinner getDurationNumberSpinner() { - if (durationNumberSpinner == null) { - durationNumberSpinner = new ZapNumberSpinner(0, 0, Integer.MAX_VALUE); - } - return durationNumberSpinner; - } - - private ZapNumberSpinner getMaxChildrenNumberSpinner() { - if (maxChildrenNumberSpinner == null) { - maxChildrenNumberSpinner = new ZapNumberSpinner(0, 0, Integer.MAX_VALUE); - } - return maxChildrenNumberSpinner; - } - - private ZapNumberSpinner getMaxParseSizeBytesNumberSpinner() { - if (maxParseSizeBytesNumberSpinner == null) { - maxParseSizeBytesNumberSpinner = new ZapNumberSpinner(0, 0, Integer.MAX_VALUE); - } - return maxParseSizeBytesNumberSpinner; - } - - private JCheckBox getChkSendRefererHeader() { - if (chkSendRefererHeader == null) { - chkSendRefererHeader = - new JCheckBox( - Constant.messages.getString("spider.options.label.sendRefererHeader")); - } - return chkSendRefererHeader; - } - - private JCheckBox getChkAcceptCookies() { - if (chkAcceptCookies == null) { - chkAcceptCookies = - new JCheckBox( - Constant.messages.getString("spider.options.label.acceptcookies")); - } - return chkAcceptCookies; - } - - /** - * This method initializes the checkbox for POST form option. This option should not be enabled - * if the forms are not processed at all. - * - * @return javax.swing.JCheckBox - */ - private JCheckBox getChkPostForm() { - if (chkPostForm == null) { - chkPostForm = new JCheckBox(); - chkPostForm.setText(Constant.messages.getString("spider.options.label.post")); - - if (!getChkProcessForm().isSelected()) { - chkPostForm.setEnabled(false); - } - } - return chkPostForm; - } - - /** - * This method initializes the checkbox for process form option. - * - * @return javax.swing.JCheckBox - */ - private JCheckBox getChkProcessForm() { - if (chkProcessForm == null) { - chkProcessForm = new JCheckBox(); - chkProcessForm.setText(Constant.messages.getString("spider.options.label.processform")); - - // Code for controlling the status of the chkPostForm - chkProcessForm.addChangeListener( - new ChangeListener() { - @Override - public void stateChanged(ChangeEvent ev) { - if (chkProcessForm.isSelected()) { - chkPostForm.setEnabled(true); - } else { - chkPostForm.setEnabled(false); - } - } - }); - } - return chkProcessForm; - } - - /** - * This method initializes the Parse Comments checkbox. - * - * @return javax.swing.JCheckBox - */ - private JCheckBox getChkParseComments() { - if (parseComments == null) { - parseComments = new JCheckBox(); - parseComments.setText(Constant.messages.getString("spider.options.label.comments")); - } - return parseComments; - } - - /** - * This method initializes the Parse robots.txt checkbox. - * - * @return javax.swing.JCheckBox - */ - private JCheckBox getChkParseRobotsTxt() { - if (parseRobotsTxt == null) { - parseRobotsTxt = new JCheckBox(); - parseRobotsTxt.setText(Constant.messages.getString("spider.options.label.robotstxt")); - } - return parseRobotsTxt; - } - - /** - * This method initializes the Parse sitemap.xml checkbox. - * - * @return javax.swing.JCheckBox - */ - private JCheckBox getChkParseSitemapXml() { - if (parseSitemapXml == null) { - parseSitemapXml = new JCheckBox(); - parseSitemapXml.setText(Constant.messages.getString("spider.options.label.sitemapxml")); - } - return parseSitemapXml; - } - - /** - * This method initializes the Parse "SVN Entries" checkbox. - * - * @return javax.swing.JCheckBox - */ - private JCheckBox getChkParseSVNEntries() { - if (parseSVNEntries == null) { - parseSVNEntries = new JCheckBox(); - parseSVNEntries.setText(Constant.messages.getString("spider.options.label.svnentries")); - } - return parseSVNEntries; - } - - /** - * This method initializes the Parse "Git" checkbox. - * - * @return javax.swing.JCheckBox - */ - private JCheckBox getChkParseGit() { - if (parseGit == null) { - parseGit = new JCheckBox(); - parseGit.setText(Constant.messages.getString("spider.options.label.git")); - } - return parseGit; - } - - /** - * This method initializes the Handle OData-specific parameters checkbox. - * - * @return javax.swing.JCheckBox - */ - private JCheckBox getHandleODataSpecificParameters() { - if (handleODataSpecificParameters == null) { - handleODataSpecificParameters = new JCheckBox(); - handleODataSpecificParameters.setText( - Constant.messages.getString("spider.options.label.handlehodataparameters")); - } - return handleODataSpecificParameters; - } - - /** - * This method initializes the combobox for HandleParameters option. - * - * @return the combo handle parameters - */ - @SuppressWarnings("unchecked") - private JComboBox - getComboHandleParameters() { - if (handleParameters == null) { - handleParameters = - new JComboBox<>( - new org.zaproxy.zap.spider.SpiderParam.HandleParametersOption[] { - org.zaproxy.zap.spider.SpiderParam.HandleParametersOption.USE_ALL, - org.zaproxy.zap.spider.SpiderParam.HandleParametersOption - .IGNORE_VALUE, - org.zaproxy.zap.spider.SpiderParam.HandleParametersOption - .IGNORE_COMPLETELY - }); - handleParameters.setRenderer(new HandleParametersOptionRenderer()); - } - return handleParameters; - } - - private DomainsAlwaysInScopeMultipleOptionsPanel getDomainsAlwaysInScopePanel() { - if (domainsAlwaysInScopePanel == null) { - domainsAlwaysInScopePanel = - new DomainsAlwaysInScopeMultipleOptionsPanel( - getDomainsAlwaysInScopeTableModel()); - } - return domainsAlwaysInScopePanel; - } - - private DomainsAlwaysInScopeTableModel getDomainsAlwaysInScopeTableModel() { - if (domainsAlwaysInScopeTableModel == null) { - domainsAlwaysInScopeTableModel = new DomainsAlwaysInScopeTableModel(); - } - return domainsAlwaysInScopeTableModel; - } - - private ZapTextArea getIrrelevantUrlParameters() { - if (irrelevantUrlParameters == null) { - irrelevantUrlParameters = new ZapTextArea(); - irrelevantUrlParameters.setLineWrap(true); - } - return irrelevantUrlParameters; - } - - /** A renderer for properly displaying the name of the HandleParametersOptions in a ComboBox. */ - private static class HandleParametersOptionRenderer extends BasicComboBoxRenderer { - private static final long serialVersionUID = 3654541772447187317L; - private static final Border BORDER = new EmptyBorder(2, 3, 3, 3); - - @Override - @SuppressWarnings("rawtypes") - public Component getListCellRendererComponent( - JList list, Object value, int index, boolean isSelected, boolean cellHasFocus) { - super.getListCellRendererComponent(list, value, index, isSelected, cellHasFocus); - if (value != null) { - setBorder(BORDER); - org.zaproxy.zap.spider.SpiderParam.HandleParametersOption item = - (org.zaproxy.zap.spider.SpiderParam.HandleParametersOption) value; - setText(item.getName()); - } - return this; - } - } - - /** - * This method initializes the help index. - * - * @return the help index - */ - @Override - public String getHelpIndex() { - return "ui.dialogs.options.spider"; - } - - private static class DomainsAlwaysInScopeMultipleOptionsPanel - extends AbstractMultipleOptionsTablePanel< - org.zaproxy.zap.spider.DomainAlwaysInScopeMatcher> { - - private static final long serialVersionUID = 2332044353650231701L; - - private static final String REMOVE_DIALOG_TITLE = - Constant.messages.getString("spider.options.domains.in.scope.dialog.remove.title"); - private static final String REMOVE_DIALOG_TEXT = - Constant.messages.getString("spider.options.domains.in.scope.dialog.remove.text"); - - private static final String REMOVE_DIALOG_CONFIRM_BUTTON_LABEL = - Constant.messages.getString( - "spider.options.domains.in.scope.dialog.remove.button.confirm"); - private static final String REMOVE_DIALOG_CANCEL_BUTTON_LABEL = - Constant.messages.getString( - "spider.options.domains.in.scope.dialog.remove.button.cancel"); - - private static final String REMOVE_DIALOG_CHECKBOX_LABEL = - Constant.messages.getString( - "spider.options.domains.in.scope.dialog.remove.checkbox.label"); - - private DialogAddDomainAlwaysInScope addDialog = null; - private DialogModifyDomainAlwaysInScope modifyDialog = null; - - public DomainsAlwaysInScopeMultipleOptionsPanel(DomainsAlwaysInScopeTableModel model) { - super(model); - - getTable().setVisibleRowCount(5); - getTable().setSortOrder(2, SortOrder.ASCENDING); - } - - @Override - public org.zaproxy.zap.spider.DomainAlwaysInScopeMatcher showAddDialogue() { - if (addDialog == null) { - addDialog = - new DialogAddDomainAlwaysInScope( - View.getSingleton().getOptionsDialog(null)); - addDialog.pack(); - } - addDialog.setVisible(true); - - org.zaproxy.zap.spider.DomainAlwaysInScopeMatcher hostAuthentication = - addDialog.getDomainAlwaysInScope(); - addDialog.clear(); - - return hostAuthentication; - } - - @Override - public org.zaproxy.zap.spider.DomainAlwaysInScopeMatcher showModifyDialogue( - org.zaproxy.zap.spider.DomainAlwaysInScopeMatcher e) { - if (modifyDialog == null) { - modifyDialog = - new DialogModifyDomainAlwaysInScope( - View.getSingleton().getOptionsDialog(null)); - modifyDialog.pack(); - } - modifyDialog.setDomainAlwaysInScope(e); - modifyDialog.setVisible(true); - - org.zaproxy.zap.spider.DomainAlwaysInScopeMatcher excludedDomain = - modifyDialog.getDomainAlwaysInScope(); - modifyDialog.clear(); - - if (!excludedDomain.equals(e)) { - return excludedDomain; - } - - return null; - } - - @Override - public boolean showRemoveDialogue(org.zaproxy.zap.spider.DomainAlwaysInScopeMatcher e) { - JCheckBox removeWithoutConfirmationCheckBox = - new JCheckBox(REMOVE_DIALOG_CHECKBOX_LABEL); - Object[] messages = {REMOVE_DIALOG_TEXT, " ", removeWithoutConfirmationCheckBox}; - int option = - JOptionPane.showOptionDialog( - View.getSingleton().getMainFrame(), - messages, - REMOVE_DIALOG_TITLE, - JOptionPane.OK_CANCEL_OPTION, - JOptionPane.QUESTION_MESSAGE, - null, - new String[] { - REMOVE_DIALOG_CONFIRM_BUTTON_LABEL, - REMOVE_DIALOG_CANCEL_BUTTON_LABEL - }, - null); - - if (option == JOptionPane.OK_OPTION) { - setRemoveWithoutConfirmation(removeWithoutConfirmationCheckBox.isSelected()); - - return true; - } - - return false; - } - } -} diff --git a/zap/src/main/java/org/zaproxy/zap/extension/spider/PopupMenuItemSpiderDialog.java b/zap/src/main/java/org/zaproxy/zap/extension/spider/PopupMenuItemSpiderDialog.java deleted file mode 100644 index 269689ff6ce..00000000000 --- a/zap/src/main/java/org/zaproxy/zap/extension/spider/PopupMenuItemSpiderDialog.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2017 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.extension.spider; - -import org.parosproxy.paros.Constant; -import org.parosproxy.paros.model.SiteNode; -import org.zaproxy.zap.view.messagecontainer.http.HttpMessageContainer; -import org.zaproxy.zap.view.popup.PopupMenuItemSiteNodeContainer; - -/** - * A {@code PopupMenuItemSiteNodeContainer} that allows to show the Spider dialogue, for a selected - * {@link SiteNode}. - * - * @see org.zaproxy.zap.extension.spider.ExtensionSpider#showSpiderDialog(SiteNode) - * @deprecated (2.12.0) See the spider add-on in zap-extensions instead. - */ -@SuppressWarnings("serial") -@Deprecated -public class PopupMenuItemSpiderDialog extends PopupMenuItemSiteNodeContainer { - - private static final long serialVersionUID = 1L; - - private final ExtensionSpider extension; - - public PopupMenuItemSpiderDialog(ExtensionSpider extension) { - super(Constant.messages.getString("spider.custom.popup")); - - this.setIcon(extension.getIcon()); - this.extension = extension; - } - - @Override - public boolean isSubMenu() { - return true; - } - - @Override - public String getParentMenuName() { - return Constant.messages.getString("attack.site.popup"); - } - - @Override - public void performAction(SiteNode node) { - extension.showSpiderDialog(node); - } - - @Override - protected boolean isEnableForInvoker( - Invoker invoker, HttpMessageContainer httpMessageContainer) { - switch (invoker) { - case ALERTS_PANEL: - case ACTIVE_SCANNER_PANEL: - case FORCED_BROWSE_PANEL: - case FUZZER_PANEL: - return false; - case HISTORY_PANEL: - case SITES_PANEL: - case SEARCH_PANEL: - default: - return true; - } - } -} diff --git a/zap/src/main/java/org/zaproxy/zap/extension/spider/PopupMenuItemSpiderDialogWithContext.java b/zap/src/main/java/org/zaproxy/zap/extension/spider/PopupMenuItemSpiderDialogWithContext.java deleted file mode 100644 index b188055d820..00000000000 --- a/zap/src/main/java/org/zaproxy/zap/extension/spider/PopupMenuItemSpiderDialogWithContext.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2018 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.extension.spider; - -import org.parosproxy.paros.model.Model; -import org.zaproxy.zap.extension.stdmenus.PopupContextTreeMenu; -import org.zaproxy.zap.model.Context; -import org.zaproxy.zap.model.Target; - -/** - * A {@code PopupContextTreeMenu} that allows to show the Spider dialogue for a selected {@link - * Context}. - * - * @see ExtensionSpider#showSpiderDialog(Target) - * @deprecated (2.12.0) See the spider add-on in zap-extensions instead. - */ -@Deprecated -public class PopupMenuItemSpiderDialogWithContext extends PopupContextTreeMenu { - - private static final long serialVersionUID = 1L; - - public PopupMenuItemSpiderDialogWithContext(ExtensionSpider extension) { - super(false); - - this.setText(extension.getMessages().getString("spider.custom.popup")); - this.setIcon(extension.getIcon()); - - this.addActionListener( - e -> { - Context context = Model.getSingleton().getSession().getContext(getContextId()); - extension.showSpiderDialog(new Target(context)); - }); - } -} diff --git a/zap/src/main/java/org/zaproxy/zap/extension/spider/SpiderAPI.java b/zap/src/main/java/org/zaproxy/zap/extension/spider/SpiderAPI.java deleted file mode 100644 index 0bcc14b27d5..00000000000 --- a/zap/src/main/java/org/zaproxy/zap/extension/spider/SpiderAPI.java +++ /dev/null @@ -1,754 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2011 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.extension.spider; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.regex.PatternSyntaxException; -import net.sf.json.JSONException; -import net.sf.json.JSONObject; -import org.apache.commons.httpclient.URI; -import org.apache.commons.httpclient.URIException; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.parosproxy.paros.control.Control; -import org.parosproxy.paros.db.DatabaseException; -import org.parosproxy.paros.db.RecordHistory; -import org.parosproxy.paros.db.TableHistory; -import org.parosproxy.paros.model.HistoryReference; -import org.parosproxy.paros.model.Model; -import org.parosproxy.paros.model.Session; -import org.parosproxy.paros.network.HttpMalformedHeaderException; -import org.zaproxy.zap.extension.api.ApiAction; -import org.zaproxy.zap.extension.api.ApiException; -import org.zaproxy.zap.extension.api.ApiException.Type; -import org.zaproxy.zap.extension.api.ApiImplementor; -import org.zaproxy.zap.extension.api.ApiResponse; -import org.zaproxy.zap.extension.api.ApiResponseElement; -import org.zaproxy.zap.extension.api.ApiResponseList; -import org.zaproxy.zap.extension.api.ApiResponseSet; -import org.zaproxy.zap.extension.api.ApiView; -import org.zaproxy.zap.extension.users.ExtensionUserManagement; -import org.zaproxy.zap.model.Context; -import org.zaproxy.zap.model.SessionStructure; -import org.zaproxy.zap.model.StructuralNode; -import org.zaproxy.zap.model.Target; -import org.zaproxy.zap.users.User; -import org.zaproxy.zap.utils.ApiUtils; - -/** - * @deprecated (2.12.0) See the spider add-on in zap-extensions instead. - */ -@Deprecated -public class SpiderAPI extends ApiImplementor { - - private static final Logger log = LogManager.getLogger(SpiderAPI.class); - - /** The Constant PREFIX defining the name/prefix of the api. */ - private static final String PREFIX = "spider"; - - /** The Constant ACTION_START_SCAN that defines the action of starting a new scan. */ - private static final String ACTION_START_SCAN = "scan"; - - private static final String ACTION_START_SCAN_AS_USER = "scanAsUser"; - - private static final String ACTION_PAUSE_SCAN = "pause"; - private static final String ACTION_RESUME_SCAN = "resume"; - - /** The Constant ACTION_STOP_SCAN that defines the action of stopping a pending scan. */ - private static final String ACTION_STOP_SCAN = "stop"; - - private static final String ACTION_PAUSE_ALL_SCANS = "pauseAllScans"; - private static final String ACTION_RESUME_ALL_SCANS = "resumeAllScans"; - private static final String ACTION_STOP_ALL_SCANS = "stopAllScans"; - private static final String ACTION_REMOVE_SCAN = "removeScan"; - private static final String ACTION_REMOVE_ALL_SCANS = "removeAllScans"; - - private static final String ACTION_ADD_DOMAIN_ALWAYS_IN_SCOPE = "addDomainAlwaysInScope"; - private static final String ACTION_MODIFY_DOMAIN_ALWAYS_IN_SCOPE = "modifyDomainAlwaysInScope"; - private static final String ACTION_REMOVE_DOMAIN_ALWAYS_IN_SCOPE = "removeDomainAlwaysInScope"; - private static final String ACTION_ENABLE_ALL_DOMAINS_ALWAYS_IN_SCOPE = - "enableAllDomainsAlwaysInScope"; - private static final String ACTION_DISABLE_ALL_DOMAINS_ALWAYS_IN_SCOPE = - "disableAllDomainsAlwaysInScope"; - - /** - * The Constant VIEW_STATUS that defines the view which describes the current status of the - * scan. - */ - private static final String VIEW_STATUS = "status"; - - /** - * The Constant VIEW_RESULTS that defines the view which describes the urls found during the - * scan. - */ - private static final String VIEW_RESULTS = "results"; - - private static final String VIEW_FULL_RESULTS = "fullResults"; - private static final String VIEW_SCANS = "scans"; - private static final String VIEW_ALL_URLS = "allUrls"; - private static final String VIEW_ADDED_NODES = "addedNodes"; - - private static final String VIEW_DOMAINS_ALWAYS_IN_SCOPE = "domainsAlwaysInScope"; - private static final String VIEW_OPTION_DOMAINS_ALWAYS_IN_SCOPE = "optionDomainsAlwaysInScope"; - private static final String VIEW_OPTION_DOMAINS_ALWAYS_IN_SCOPE_ENABLED = - "optionDomainsAlwaysInScopeEnabled"; - - /** The Constant PARAM_URL that defines the parameter defining the url of the scan. */ - private static final String PARAM_URL = "url"; - - private static final String PARAM_USER_ID = "userId"; - private static final String PARAM_CONTEXT_ID = "contextId"; - private static final String PARAM_CONTEXT_NAME = "contextName"; - private static final String PARAM_REGEX = "regex"; - private static final String PARAM_RECURSE = "recurse"; - private static final String PARAM_SCAN_ID = "scanId"; - private static final String PARAM_MAX_CHILDREN = "maxChildren"; - private static final String PARAM_SUBTREE_ONLY = "subtreeOnly"; - private static final String PARAM_VALUE = "value"; - private static final String PARAM_IDX = "idx"; - private static final String PARAM_IS_REGEX = "isRegex"; - private static final String PARAM_IS_ENABLED = "isEnabled"; - - private static final String ACTION_EXCLUDE_FROM_SCAN = "excludeFromScan"; - private static final String ACTION_CLEAR_EXCLUDED_FROM_SCAN = "clearExcludedFromScan"; - - private static final String VIEW_EXCLUDED_FROM_SCAN = "excludedFromScan"; - - /** The spider extension. */ - private ExtensionSpider extension; - - /** - * Instantiates a new spider API. - * - * @param extension the extension - */ - public SpiderAPI(ExtensionSpider extension) { - this.extension = extension; - // Register the actions - this.addApiAction( - new ApiAction( - ACTION_START_SCAN, - null, - new String[] { - PARAM_URL, - PARAM_MAX_CHILDREN, - PARAM_RECURSE, - PARAM_CONTEXT_NAME, - PARAM_SUBTREE_ONLY - })); - this.addApiAction( - new ApiAction( - ACTION_START_SCAN_AS_USER, - new String[] {PARAM_CONTEXT_ID, PARAM_USER_ID}, - new String[] { - PARAM_URL, PARAM_MAX_CHILDREN, PARAM_RECURSE, PARAM_SUBTREE_ONLY - })); - this.addApiAction(new ApiAction(ACTION_PAUSE_SCAN, new String[] {PARAM_SCAN_ID})); - this.addApiAction(new ApiAction(ACTION_RESUME_SCAN, new String[] {PARAM_SCAN_ID})); - this.addApiAction(new ApiAction(ACTION_STOP_SCAN, null, new String[] {PARAM_SCAN_ID})); - this.addApiAction(new ApiAction(ACTION_REMOVE_SCAN, new String[] {PARAM_SCAN_ID})); - this.addApiAction(new ApiAction(ACTION_PAUSE_ALL_SCANS)); - this.addApiAction(new ApiAction(ACTION_RESUME_ALL_SCANS)); - this.addApiAction(new ApiAction(ACTION_STOP_ALL_SCANS)); - this.addApiAction(new ApiAction(ACTION_REMOVE_ALL_SCANS)); - this.addApiAction(new ApiAction(ACTION_CLEAR_EXCLUDED_FROM_SCAN)); - this.addApiAction(new ApiAction(ACTION_EXCLUDE_FROM_SCAN, new String[] {PARAM_REGEX})); - - this.addApiAction( - new ApiAction( - ACTION_ADD_DOMAIN_ALWAYS_IN_SCOPE, - new String[] {PARAM_VALUE}, - new String[] {PARAM_IS_REGEX, PARAM_IS_ENABLED})); - this.addApiAction( - new ApiAction( - ACTION_MODIFY_DOMAIN_ALWAYS_IN_SCOPE, - new String[] {PARAM_IDX}, - new String[] {PARAM_VALUE, PARAM_IS_REGEX, PARAM_IS_ENABLED})); - this.addApiAction( - new ApiAction(ACTION_REMOVE_DOMAIN_ALWAYS_IN_SCOPE, new String[] {PARAM_IDX})); - this.addApiAction(new ApiAction(ACTION_ENABLE_ALL_DOMAINS_ALWAYS_IN_SCOPE)); - this.addApiAction(new ApiAction(ACTION_DISABLE_ALL_DOMAINS_ALWAYS_IN_SCOPE)); - - // Register the views - this.addApiView(new ApiView(VIEW_STATUS, null, new String[] {PARAM_SCAN_ID})); - this.addApiView(new ApiView(VIEW_RESULTS, null, new String[] {PARAM_SCAN_ID})); - this.addApiView(new ApiView(VIEW_FULL_RESULTS, new String[] {PARAM_SCAN_ID})); - this.addApiView(new ApiView(VIEW_SCANS)); - this.addApiView(new ApiView(VIEW_EXCLUDED_FROM_SCAN)); - this.addApiView(new ApiView(VIEW_ALL_URLS)); - this.addApiView(new ApiView(VIEW_ADDED_NODES, null, new String[] {PARAM_SCAN_ID})); - - this.addApiView(new ApiView(VIEW_DOMAINS_ALWAYS_IN_SCOPE)); - ApiView view = new ApiView(VIEW_OPTION_DOMAINS_ALWAYS_IN_SCOPE); - view.setDeprecated(true); - this.addApiView(view); - view = new ApiView(VIEW_OPTION_DOMAINS_ALWAYS_IN_SCOPE_ENABLED); - view.setDeprecated(true); - this.addApiView(view); - } - - @Override - public String getPrefix() { - return PREFIX; - } - - @Override - public ApiResponse handleApiAction(String name, JSONObject params) throws ApiException { - log.debug("Request for handleApiAction: {} (params: {})", name, params); - SpiderScan scan; - int maxChildren = -1; - Context context = null; - - switch (name) { - case ACTION_START_SCAN: - // The action is to start a new Scan - String url = ApiUtils.getOptionalStringParam(params, PARAM_URL); - if (params.containsKey(PARAM_MAX_CHILDREN)) { - String maxChildrenStr = params.getString(PARAM_MAX_CHILDREN); - if (maxChildrenStr != null && maxChildrenStr.length() > 0) { - try { - maxChildren = Integer.parseInt(maxChildrenStr); - } catch (NumberFormatException e) { - throw new ApiException(Type.ILLEGAL_PARAMETER, PARAM_MAX_CHILDREN); - } - } - } - if (params.containsKey(PARAM_CONTEXT_NAME)) { - String contextName = params.getString(PARAM_CONTEXT_NAME); - if (!contextName.isEmpty()) { - context = ApiUtils.getContextByName(contextName); - } - } - int scanId = - scanURL( - url, - null, - maxChildren, - this.getParam(params, PARAM_RECURSE, true), - context, - getParam(params, PARAM_SUBTREE_ONLY, false)); - return new ApiResponseElement(name, Integer.toString(scanId)); - - case ACTION_START_SCAN_AS_USER: - // The action is to start a new Scan from the perspective of a user - String urlUserScan = ApiUtils.getOptionalStringParam(params, PARAM_URL); - int userID = ApiUtils.getIntParam(params, PARAM_USER_ID); - ExtensionUserManagement usersExtension = - Control.getSingleton() - .getExtensionLoader() - .getExtension(ExtensionUserManagement.class); - if (usersExtension == null) { - throw new ApiException(Type.NO_IMPLEMENTOR, ExtensionUserManagement.NAME); - } - context = ApiUtils.getContextByParamId(params, PARAM_CONTEXT_ID); - User user = - usersExtension - .getContextUserAuthManager(context.getId()) - .getUserById(userID); - if (user == null) { - throw new ApiException(Type.USER_NOT_FOUND, PARAM_USER_ID); - } - if (params.containsKey(PARAM_MAX_CHILDREN)) { - String maxChildrenStr = params.getString(PARAM_MAX_CHILDREN); - if (maxChildrenStr != null && maxChildrenStr.length() > 0) { - try { - maxChildren = Integer.parseInt(maxChildrenStr); - } catch (NumberFormatException e) { - throw new ApiException(Type.ILLEGAL_PARAMETER, PARAM_MAX_CHILDREN); - } - } - } - scanId = - scanURL( - urlUserScan, - user, - maxChildren, - this.getParam(params, PARAM_RECURSE, true), - context, - getParam(params, PARAM_SUBTREE_ONLY, false)); - - return new ApiResponseElement(name, Integer.toString(scanId)); - - case ACTION_PAUSE_SCAN: - scan = getSpiderScan(params); - extension.pauseScan(scan.getScanId()); - break; - case ACTION_RESUME_SCAN: - scan = getSpiderScan(params); - extension.resumeScan(scan.getScanId()); - break; - case ACTION_STOP_SCAN: - // The action is to stop a pending scan - scan = getSpiderScan(params); - extension.stopScan(scan.getScanId()); - break; - case ACTION_REMOVE_SCAN: - // Note that we're removing the scan with this call, not just getting it ;) - scan = getSpiderScan(params); - extension.removeScan(scan.getScanId()); - break; - case ACTION_PAUSE_ALL_SCANS: - extension.pauseAllScans(); - break; - case ACTION_RESUME_ALL_SCANS: - extension.resumeAllScans(); - break; - case ACTION_STOP_ALL_SCANS: - extension.stopAllScans(); - break; - case ACTION_REMOVE_ALL_SCANS: - extension.removeAllScans(); - break; - case ACTION_CLEAR_EXCLUDED_FROM_SCAN: - try { - Session session = Model.getSingleton().getSession(); - session.setExcludeFromSpiderRegexs(new ArrayList<>()); - } catch (DatabaseException e) { - throw new ApiException(ApiException.Type.INTERNAL_ERROR, e.getMessage()); - } - break; - case ACTION_EXCLUDE_FROM_SCAN: - String regex = params.getString(PARAM_REGEX); - try { - Session session = Model.getSingleton().getSession(); - session.addExcludeFromSpiderRegex(regex); - } catch (DatabaseException e) { - throw new ApiException(ApiException.Type.INTERNAL_ERROR, e.getMessage()); - } catch (PatternSyntaxException e) { - throw new ApiException(ApiException.Type.ILLEGAL_PARAMETER, PARAM_REGEX); - } - break; - - case ACTION_ADD_DOMAIN_ALWAYS_IN_SCOPE: - try { - String value = params.getString(PARAM_VALUE); - org.zaproxy.zap.spider.DomainAlwaysInScopeMatcher domainAlwaysInScope; - if (getParam(params, PARAM_IS_REGEX, false)) { - domainAlwaysInScope = - new org.zaproxy.zap.spider.DomainAlwaysInScopeMatcher( - org.zaproxy.zap.spider.DomainAlwaysInScopeMatcher - .createPattern(value)); - } else { - domainAlwaysInScope = - new org.zaproxy.zap.spider.DomainAlwaysInScopeMatcher(value); - } - domainAlwaysInScope.setEnabled(getParam(params, PARAM_IS_ENABLED, true)); - - List domainsAlwaysInScope = - new ArrayList<>(extension.getSpiderParam().getDomainsAlwaysInScope()); - domainsAlwaysInScope.add(domainAlwaysInScope); - extension.getSpiderParam().setDomainsAlwaysInScope(domainsAlwaysInScope); - } catch (IllegalArgumentException e) { - throw new ApiException(ApiException.Type.ILLEGAL_PARAMETER, PARAM_VALUE, e); - } - break; - case ACTION_MODIFY_DOMAIN_ALWAYS_IN_SCOPE: - try { - int idx = params.getInt(PARAM_IDX); - if (idx < 0 - || idx >= extension.getSpiderParam().getDomainsAlwaysInScope().size()) { - throw new ApiException(ApiException.Type.ILLEGAL_PARAMETER, PARAM_IDX); - } - - org.zaproxy.zap.spider.DomainAlwaysInScopeMatcher oldDomain = - extension.getSpiderParam().getDomainsAlwaysInScope().get(idx); - String value = getParam(params, PARAM_VALUE, oldDomain.getValue()); - if (value.isEmpty()) { - value = oldDomain.getValue(); - } - - org.zaproxy.zap.spider.DomainAlwaysInScopeMatcher newDomain; - if (getParam(params, PARAM_IS_REGEX, oldDomain.isRegex())) { - newDomain = - new org.zaproxy.zap.spider.DomainAlwaysInScopeMatcher( - org.zaproxy.zap.spider.DomainAlwaysInScopeMatcher - .createPattern(value)); - } else { - newDomain = new org.zaproxy.zap.spider.DomainAlwaysInScopeMatcher(value); - } - newDomain.setEnabled(getParam(params, PARAM_IS_ENABLED, oldDomain.isEnabled())); - - if (oldDomain.equals(newDomain)) { - break; - } - - List domainsAlwaysInScope = - new ArrayList<>(extension.getSpiderParam().getDomainsAlwaysInScope()); - domainsAlwaysInScope.set(idx, newDomain); - extension.getSpiderParam().setDomainsAlwaysInScope(domainsAlwaysInScope); - } catch (JSONException e) { - throw new ApiException(ApiException.Type.ILLEGAL_PARAMETER, PARAM_IDX, e); - } catch (IllegalArgumentException e) { - throw new ApiException(ApiException.Type.ILLEGAL_PARAMETER, PARAM_VALUE, e); - } - break; - case ACTION_REMOVE_DOMAIN_ALWAYS_IN_SCOPE: - try { - int idx = params.getInt(PARAM_IDX); - if (idx < 0 - || idx >= extension.getSpiderParam().getDomainsAlwaysInScope().size()) { - throw new ApiException(ApiException.Type.ILLEGAL_PARAMETER, PARAM_IDX); - } - - List domainsAlwaysInScope = - new ArrayList<>(extension.getSpiderParam().getDomainsAlwaysInScope()); - domainsAlwaysInScope.remove(idx); - extension.getSpiderParam().setDomainsAlwaysInScope(domainsAlwaysInScope); - } catch (JSONException e) { - throw new ApiException(ApiException.Type.ILLEGAL_PARAMETER, PARAM_IDX, e); - } - break; - case ACTION_ENABLE_ALL_DOMAINS_ALWAYS_IN_SCOPE: - setDomainsAlwaysInScopeEnabled(true); - break; - case ACTION_DISABLE_ALL_DOMAINS_ALWAYS_IN_SCOPE: - setDomainsAlwaysInScopeEnabled(false); - break; - default: - throw new ApiException(ApiException.Type.BAD_ACTION); - } - return ApiResponseElement.OK; - } - - private void setDomainsAlwaysInScopeEnabled(boolean enabled) { - List domainsAlwaysInScope = - extension.getSpiderParam().getDomainsAlwaysInScope(); - for (org.zaproxy.zap.spider.DomainAlwaysInScopeMatcher x : - extension.getSpiderParam().getDomainsAlwaysInScope()) { - x.setEnabled(enabled); - } - extension.getSpiderParam().setDomainsAlwaysInScope(domainsAlwaysInScope); - } - - /** - * Returns the specified GenericScanner2 or the last scan available. - * - * @param params the parameters of the API call - * @return the GenericScanner2 with the given scan ID or, if not present, the last scan - * available - * @throws ApiException if there's no scan with the given scan ID - * @see #PARAM_SCAN_ID - */ - private SpiderScan getSpiderScan(JSONObject params) throws ApiException { - SpiderScan spiderScan; - int id = getParam(params, PARAM_SCAN_ID, -1); - if (id == -1) { - spiderScan = extension.getLastScan(); - } else { - spiderScan = extension.getScan(id); - } - - if (spiderScan == null) { - throw new ApiException(ApiException.Type.DOES_NOT_EXIST, PARAM_SCAN_ID); - } - - return spiderScan; - } - - /** - * Starts a spider scan at the given {@code url} and, optionally, with the perspective of the - * given {@code user}. - * - * @param url the url to start the spider scan - * @param user the user to scan as, or null if the scan is done without the perspective of any - * user - * @param maxChildren Max number of children to scan - * @param recurse Whether or not to scan recursively - * @param context the context that will be used during spider process, might be {@code null} - * @param subtreeOnly if the scan should be done only under a site's subtree - * @return the ID of the newly started scan - * @throws ApiException if the {@code url} is not valid - */ - private int scanURL( - String url, - User user, - int maxChildren, - boolean recurse, - Context context, - boolean subtreeOnly) - throws ApiException { - log.debug("API Spider scanning url: {}", url); - - boolean useUrl = true; - if (url == null || url.isEmpty()) { - if (context == null || !context.hasNodesInContextFromSiteTree()) { - throw new ApiException(Type.MISSING_PARAMETER, PARAM_URL); - } - useUrl = false; - } else if (context != null && !context.isInContext(url)) { - throw new ApiException(Type.URL_NOT_IN_CONTEXT, PARAM_URL); - } - - StructuralNode node = null; - URI startURI = null; - if (useUrl) { - try { - // Try to build uri - startURI = new URI(url, true); - } catch (URIException e) { - throw new ApiException(ApiException.Type.ILLEGAL_PARAMETER, PARAM_URL); - } - String scheme = startURI.getScheme(); - if (scheme == null - || (!scheme.equalsIgnoreCase("http") && !scheme.equalsIgnoreCase("https"))) { - throw new ApiException(ApiException.Type.ILLEGAL_PARAMETER, PARAM_URL); - } - - node = getStartNode(startURI, recurse); - } - Target target = new Target(); - if (useUrl && node != null) { - target.setStartNode(node); - } - target.setContext(context); - target.setRecurse(recurse); - - switch (Control.getSingleton().getMode()) { - case safe: - throw new ApiException(ApiException.Type.MODE_VIOLATION); - case protect: - if ((useUrl && !Model.getSingleton().getSession().isInScope(url)) - || (context != null && !context.isInScope())) { - throw new ApiException(ApiException.Type.MODE_VIOLATION); - } - // No problem - break; - case standard: - // No problem - break; - case attack: - // No problem - break; - } - - List objs = new ArrayList<>(4); - if (startURI != null) { - objs.add(startURI); - if (subtreeOnly) { - objs.add(new org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter(startURI)); - } - } - - if (maxChildren > 0) { - // Add the filters to filter on maximum number of children - org.zaproxy.zap.spider.filters.MaxChildrenFetchFilter maxChildrenFetchFilter = - new org.zaproxy.zap.spider.filters.MaxChildrenFetchFilter(); - maxChildrenFetchFilter.setMaxChildren(maxChildren); - maxChildrenFetchFilter.setModel(extension.getModel()); - - org.zaproxy.zap.spider.filters.MaxChildrenParseFilter maxChildrenParseFilter = - new org.zaproxy.zap.spider.filters.MaxChildrenParseFilter( - extension.getMessages()); - maxChildrenParseFilter.setMaxChildren(maxChildren); - maxChildrenParseFilter.setModel(extension.getModel()); - objs.add(maxChildrenFetchFilter); - objs.add(maxChildrenParseFilter); - } - - return extension.startScan(target, user, objs.toArray(new Object[objs.size()])); - } - - /** - * Gets a node for the given URI and recurse value, for use as start/seed of the spider. - * - *

If {@code recurse} is {@code true} it returns a node that represents a directory (if - * present in the session) otherwise the node represents a GET node of the URI. If neither of - * the nodes is found in the session it returns {@code null}. - * - * @param startURI the starting URI, must not be {@code null}. - * @param recurse {@code true} if the spider should use existing URLs as seeds, {@code false} - * otherwise. - * @return the start node, might be {@code null}. - * @throws ApiException if an error occurred while obtaining the start node. - */ - private StructuralNode getStartNode(URI startURI, boolean recurse) throws ApiException { - StructuralNode startNode = null; - try { - if (recurse) { - startNode = SessionStructure.find(Model.getSingleton(), startURI, "", ""); - } - if (startNode == null) { - startNode = SessionStructure.find(Model.getSingleton(), startURI, "GET", ""); - } - } catch (Exception e) { - throw new ApiException(ApiException.Type.INTERNAL_ERROR, e); - } - return startNode; - } - - @Override - public ApiResponse handleApiView(String name, JSONObject params) throws ApiException { - ApiResponse result; - if (VIEW_STATUS.equals(name)) { - SpiderScan scan = this.getSpiderScan(params); - int progress = 0; - if (scan.isStopped()) { - progress = 100; - } else { - progress = scan.getProgress(); - } - result = new ApiResponseElement(name, Integer.toString(progress)); - } else if (VIEW_RESULTS.equals(name)) { - result = new ApiResponseList(name); - SpiderScan scan = this.getSpiderScan(params); - synchronized (scan.getResults()) { - for (String s : scan.getResults()) { - ((ApiResponseList) result).addItem(new ApiResponseElement("url", s)); - } - } - } else if (VIEW_FULL_RESULTS.equals(name)) { - ApiResponseList resultUrls = new ApiResponseList(name); - SpiderScan scan = this.getSpiderScan(params); - ApiResponseList resultList = new ApiResponseList("urlsInScope"); - synchronized (scan.getResourcesFound()) { - for (SpiderResource sr : scan.getResourcesFound()) { - resultList.addItem( - createApiResponseSet(sr, sr.isProcessed(), sr.getReasonNotProcessed())); - } - } - resultUrls.addItem(resultList); - - resultList = new ApiResponseList("urlsOutOfScope"); - synchronized (scan.getResultsOutOfScope()) { - for (String url : scan.getResultsOutOfScope()) { - resultList.addItem(new ApiResponseElement("url", url)); - } - } - resultUrls.addItem(resultList); - - resultList = new ApiResponseList("urlsIoError"); - synchronized (scan.getResourcesIoErrors()) { - for (SpiderResource sr : scan.getResourcesIoErrors()) { - resultList.addItem( - createApiResponseSet(sr, sr.isProcessed(), sr.getReasonNotProcessed())); - } - } - resultUrls.addItem(resultList); - result = resultUrls; - } else if (VIEW_EXCLUDED_FROM_SCAN.equals(name)) { - result = new ApiResponseList(name); - Session session = Model.getSingleton().getSession(); - List regexs = session.getExcludeFromSpiderRegexs(); - for (String regex : regexs) { - ((ApiResponseList) result).addItem(new ApiResponseElement("regex", regex)); - } - } else if (VIEW_SCANS.equals(name)) { - ApiResponseList resultList = new ApiResponseList(name); - for (SpiderScan spiderScan : extension.getAllScans()) { - Map map = new HashMap<>(); - map.put("id", Integer.toString(spiderScan.getScanId())); - map.put("progress", Integer.toString(spiderScan.getProgress())); - map.put("state", spiderScan.getState()); - resultList.addItem(new ApiResponseSet<>("scan", map)); - } - result = resultList; - } else if (VIEW_ALL_URLS.equals(name)) { - ApiResponseList resultUrls = new ApiResponseList(name); - Set urlSet = new HashSet<>(); - - TableHistory tableHistory = extension.getModel().getDb().getTableHistory(); - List ids = Collections.emptyList(); - - try { - ids = - tableHistory.getHistoryIdsOfHistType( - extension.getModel().getSession().getSessionId(), - HistoryReference.TYPE_SPIDER, - HistoryReference.TYPE_SPIDER_TASK); - } catch (DatabaseException e) { - throw new ApiException(ApiException.Type.INTERNAL_ERROR, e.getMessage()); - } - - String url; - for (Integer id : ids) { - try { - RecordHistory rh = tableHistory.read(id); - if (rh != null) { - url = rh.getHttpMessage().getRequestHeader().getURI().toString(); - if (urlSet.add(url)) { - resultUrls.addItem(new ApiResponseElement("url", url)); - } - } - } catch (HttpMalformedHeaderException | DatabaseException e) { - throw new ApiException(ApiException.Type.INTERNAL_ERROR, e.getMessage()); - } - } - - result = resultUrls; - } else if (VIEW_ADDED_NODES.equals(name)) { - result = new ApiResponseList(name); - SpiderScan scan = this.getSpiderScan(params); - for (String s : scan.getAddedNodesTableModel().getAddedNodes()) { - ((ApiResponseList) result).addItem(new ApiResponseElement("url", s)); - } - } else if (VIEW_DOMAINS_ALWAYS_IN_SCOPE.equals(name) - || VIEW_OPTION_DOMAINS_ALWAYS_IN_SCOPE.equals(name)) { - result = - domainMatchersToApiResponseList( - name, extension.getSpiderParam().getDomainsAlwaysInScope(), false); - } else if (VIEW_OPTION_DOMAINS_ALWAYS_IN_SCOPE_ENABLED.equals(name)) { - result = - domainMatchersToApiResponseList( - name, extension.getSpiderParam().getDomainsAlwaysInScope(), true); - } else { - throw new ApiException(ApiException.Type.BAD_VIEW); - } - return result; - } - - private static ApiResponseSet createApiResponseSet( - SpiderResource sr, boolean processed, String reasonNotProcessed) { - Map map = new HashMap<>(); - map.put("messageId", Integer.toString(sr.getHistoryId())); - map.put("method", sr.getMethod()); - map.put("url", sr.getUri()); - map.put("statusCode", Integer.toString(sr.getStatusCode())); - map.put("statusReason", sr.getStatusReason()); - map.put("processed", Boolean.toString(processed)); - map.put("reasonNotProcessed", reasonNotProcessed); - return new ApiResponseSet<>("resource", map); - } - - private ApiResponse domainMatchersToApiResponseList( - String name, - List domains, - boolean excludeDisabled) { - ApiResponseList apiResponse = new ApiResponseList(name); - for (int i = 0; i < domains.size(); i++) { - org.zaproxy.zap.spider.DomainAlwaysInScopeMatcher domain = domains.get(i); - if (!domain.isEnabled() && excludeDisabled) { - continue; - } - Map domainData = new HashMap<>(); - domainData.put("idx", i); - domainData.put("value", domain.getValue()); - domainData.put("regex", domain.isRegex()); - domainData.put("enabled", domain.isEnabled()); - apiResponse.addItem(new ApiResponseSet<>("domain", domainData)); - } - return apiResponse; - } -} diff --git a/zap/src/main/java/org/zaproxy/zap/extension/spider/SpiderDialog.java b/zap/src/main/java/org/zaproxy/zap/extension/spider/SpiderDialog.java deleted file mode 100644 index 16d715d9b22..00000000000 --- a/zap/src/main/java/org/zaproxy/zap/extension/spider/SpiderDialog.java +++ /dev/null @@ -1,459 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2014 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.extension.spider; - -import java.awt.Dimension; -import java.awt.Frame; -import java.awt.event.ActionEvent; -import java.awt.event.ActionListener; -import java.net.URL; -import java.util.ArrayList; -import java.util.List; -import javax.swing.JButton; -import org.apache.commons.httpclient.URI; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.parosproxy.paros.Constant; -import org.parosproxy.paros.control.Control; -import org.parosproxy.paros.model.Model; -import org.parosproxy.paros.model.Session; -import org.zaproxy.zap.extension.users.ExtensionUserManagement; -import org.zaproxy.zap.model.Context; -import org.zaproxy.zap.model.StructuralNode; -import org.zaproxy.zap.model.StructuralSiteNode; -import org.zaproxy.zap.model.Target; -import org.zaproxy.zap.users.User; -import org.zaproxy.zap.view.StandardFieldsDialog; - -@SuppressWarnings("serial") -/** - * @deprecated (2.12.0) See the spider add-on in zap-extensions instead. - */ -@Deprecated -public class SpiderDialog extends StandardFieldsDialog { - - private static final String FIELD_START = "spider.custom.label.start"; - private static final String FIELD_CONTEXT = "spider.custom.label.context"; - private static final String FIELD_USER = "spider.custom.label.user"; - private static final String FIELD_RECURSE = "spider.custom.label.recurse"; - private static final String FIELD_SUBTREE_ONLY = "spider.custom.label.spiderSubtreeOnly"; - private static final String FIELD_ADVANCED = "spider.custom.label.adv"; - private static final String FIELD_MAX_DEPTH = "spider.custom.label.maxDepth"; - private static final String FIELD_MAX_CHILDREN = "spider.custom.label.maxChildren"; - private static final String FIELD_MAX_DURATION = "spider.custom.label.maxDuration"; - private static final String FIELD_MAX_PARSE_SIZE_BYTES = - "spider.custom.label.maxParseSizeBytes"; - private static final String FIELD_SEND_REFERER = "spider.custom.label.sendReferer"; - private static final String FIELD_ACCEPT_COOKIES = "spider.custom.label.acceptcookies"; - private static final String FIELD_PROCESS_FORMS = "spider.custom.label.processForms"; - private static final String FIELD_POST_FORMS = "spider.custom.label.postForms"; - private static final String FIELD_PARSE_COMMENTS = "spider.custom.label.parseComments"; - private static final String FIELD_PARSE_ROBOTS = "spider.custom.label.parseRobots"; - private static final String FIELD_PARSE_SITEMAP = "spider.custom.label.sitemap"; - private static final String FIELD_PARSE_SVN = "spider.custom.label.parseSvn"; - private static final String FIELD_PARSE_GIT = "spider.custom.label.parseGit"; - private static final String FIELD_HANDLE_ODATA = "spider.custom.label.handleOdata"; - private static final String FIELD_IRRELEVANT_URL_PARAMETERS = - "spider.custom.label.irrelevantUrlParameters"; - - private static Logger logger = LogManager.getLogger(SpiderDialog.class); - - private static final long serialVersionUID = 1L; - - private JButton[] extraButtons = null; - - private ExtensionSpider extension = null; - private org.zaproxy.zap.spider.SpiderParam spiderParam = null; - - /** - * Flag that holds the previous checked state of the "Subtree Only" checkbox. - * - *

Used to restore the previous checked state between dialogue invocations. - * - * @see #FIELD_SUBTREE_ONLY - */ - private boolean subtreeOnlyPreviousCheckedState; - - private ExtensionUserManagement extUserMgmt = - Control.getSingleton().getExtensionLoader().getExtension(ExtensionUserManagement.class); - - private Target target = null; - - public SpiderDialog(ExtensionSpider ext, Frame owner, Dimension dim) { - super( - owner, - "spider.custom.title", - dim, - new String[] {"spider.custom.tab.scope", "spider.custom.tab.adv"}); - this.extension = ext; - - // The first time init to the default options set, after that keep own copies - reset(false); - } - - public void init(Target target) { - if (target != null) { - // If one isn't specified then leave the previously selected one - this.target = target; - } - logger.debug("init {}", this.target); - - this.removeAllFields(); - - this.addTargetSelectField(0, FIELD_START, this.target, true, false); - this.addComboField(0, FIELD_CONTEXT, new String[] {}, ""); - this.addComboField(0, FIELD_USER, new String[] {}, ""); - setUsers(); - this.addCheckBoxField(0, FIELD_RECURSE, true); - this.addCheckBoxField(0, FIELD_SUBTREE_ONLY, subtreeOnlyPreviousCheckedState); - // This option is always read from the 'global' options - this.addCheckBoxField(0, FIELD_ADVANCED, extension.getSpiderParam().isShowAdvancedDialog()); - this.addPadding(0); - - // Advanced options - this.addNumberField(1, FIELD_MAX_DEPTH, 0, 19, getSpiderParam().getMaxDepth()); - this.addNumberField( - 1, FIELD_MAX_CHILDREN, 0, Integer.MAX_VALUE, getSpiderParam().getMaxChildren()); - this.addNumberField( - 1, FIELD_MAX_DURATION, 0, Integer.MAX_VALUE, getSpiderParam().getMaxDuration()); - this.addNumberField( - 1, - FIELD_MAX_PARSE_SIZE_BYTES, - 0, - Integer.MAX_VALUE, - getSpiderParam().getMaxParseSizeBytes()); - this.addCheckBoxField(1, FIELD_SEND_REFERER, getSpiderParam().isSendRefererHeader()); - this.addCheckBoxField(1, FIELD_ACCEPT_COOKIES, getSpiderParam().isAcceptCookies()); - this.addCheckBoxField(1, FIELD_PROCESS_FORMS, getSpiderParam().isProcessForm()); - this.addCheckBoxField(1, FIELD_POST_FORMS, getSpiderParam().isPostForm()); - this.addCheckBoxField(1, FIELD_PARSE_COMMENTS, getSpiderParam().isParseComments()); - this.addCheckBoxField(1, FIELD_PARSE_ROBOTS, getSpiderParam().isParseRobotsTxt()); - this.addCheckBoxField(1, FIELD_PARSE_SITEMAP, getSpiderParam().isParseSitemapXml()); - this.addCheckBoxField(1, FIELD_PARSE_SVN, getSpiderParam().isParseSVNEntries()); - this.addCheckBoxField(1, FIELD_PARSE_GIT, getSpiderParam().isParseGit()); - this.addCheckBoxField( - 1, FIELD_HANDLE_ODATA, getSpiderParam().isHandleODataParametersVisited()); - this.addMultilineField( - 1, - FIELD_IRRELEVANT_URL_PARAMETERS, - getSpiderParam().getIrrelevantUrlParametersAsString()); - this.addPadding(1); - - if (!getBoolValue(FIELD_PROCESS_FORMS)) { - setFieldValue(FIELD_POST_FORMS, false); - getField(FIELD_POST_FORMS).setEnabled(false); - } - - this.addFieldListener( - FIELD_CONTEXT, - new ActionListener() { - @Override - public void actionPerformed(ActionEvent e) { - setUsers(); - } - }); - this.addFieldListener( - FIELD_PROCESS_FORMS, - new ActionListener() { - @Override - public void actionPerformed(ActionEvent e) { - if (getBoolValue(FIELD_PROCESS_FORMS)) { - getField(FIELD_POST_FORMS).setEnabled(true); - } else { - setFieldValue(FIELD_POST_FORMS, false); - getField(FIELD_POST_FORMS).setEnabled(false); - } - } - }); - this.addFieldListener( - FIELD_ADVANCED, - new ActionListener() { - @Override - public void actionPerformed(ActionEvent e) { - setAdvancedTabs(getBoolValue(FIELD_ADVANCED)); - } - }); - - if (target != null) { - // Set up the fields if a node has been specified, otherwise leave as previously set - this.targetSelected(FIELD_START, this.target); - this.setUsers(); - } - - if (!extension.getSpiderParam().isShowAdvancedDialog()) { - // Remove all but the first tab - this.setAdvancedTabs(false); - } - - setTabScrollable("spider.custom.tab.adv", true); - - this.pack(); - } - - private org.zaproxy.zap.spider.SpiderParam getSpiderParam() { - if (spiderParam == null) { - // First time in clone the global options, after that keep the last ones the user set - spiderParam = (org.zaproxy.zap.spider.SpiderParam) extension.getSpiderParam().clone(); - } - return spiderParam; - } - - private void setAdvancedTabs(boolean visible) { - // Show/hide all except from the first tab - this.setTabsVisible(new String[] {"spider.custom.tab.adv"}, visible); - } - - @Override - public String getHelpIndex() { - return "ui.dialogs.spider"; - } - - @Override - public void targetSelected(String field, Target node) { - List ctxNames = new ArrayList<>(); - if (node != null) { - // The user has selected a new node - this.target = node; - if (node.getStartNode() != null) { - Session session = Model.getSingleton().getSession(); - List contexts = session.getContextsForNode(node.getStartNode()); - for (Context context : contexts) { - ctxNames.add(context.getName()); - } - - } else if (node.getContext() != null) { - ctxNames.add(node.getContext().getName()); - } - } - this.setComboFields(FIELD_CONTEXT, ctxNames, ""); - this.getField(FIELD_CONTEXT).setEnabled(ctxNames.size() > 0); - } - - private Context getSelectedContext() { - String ctxName = this.getStringValue(FIELD_CONTEXT); - if (this.extUserMgmt != null && !this.isEmptyField(FIELD_CONTEXT)) { - Session session = Model.getSingleton().getSession(); - return session.getContext(ctxName); - } - return null; - } - - private User getSelectedUser() { - Context context = this.getSelectedContext(); - if (context != null) { - String userName = this.getStringValue(FIELD_USER); - List users = - this.extUserMgmt.getContextUserAuthManager(context.getId()).getUsers(); - for (User user : users) { - if (userName.equals(user.getName())) { - return user; - } - } - } - return null; - } - - private void setUsers() { - Context context = this.getSelectedContext(); - List userNames = new ArrayList<>(); - if (context != null) { - List users = - this.extUserMgmt.getContextUserAuthManager(context.getId()).getUsers(); - userNames.add(""); // The default should always be 'not specified' - for (User user : users) { - userNames.add(user.getName()); - } - } - this.setComboFields(FIELD_USER, userNames, ""); - this.getField(FIELD_USER).setEnabled(userNames.size() > 1); // Theres always 1.. - } - - private void reset(boolean refreshUi) { - // Reset to the global options - spiderParam = null; - subtreeOnlyPreviousCheckedState = false; - - if (refreshUi) { - init(target); - repaint(); - } - } - - @Override - public String getSaveButtonText() { - return Constant.messages.getString("spider.custom.button.scan"); - } - - @Override - public JButton[] getExtraButtons() { - if (extraButtons == null) { - JButton resetButton = - new JButton(Constant.messages.getString("spider.custom.button.reset")); - resetButton.addActionListener( - new java.awt.event.ActionListener() { - @Override - public void actionPerformed(java.awt.event.ActionEvent e) { - reset(true); - } - }); - - extraButtons = new JButton[] {resetButton}; - } - return extraButtons; - } - - @Override - public void save() { - List contextSpecificObjects = new ArrayList<>(); - URI startUri = null; - try { - // Always include the startUri, this has the side effect - // of handling URLs that have not been accessed - startUri = new URI(this.getStringValue(FIELD_START), true); - } catch (Exception e1) { - // Ignore - will have been checked in validateParams - } - if (this.getBoolValue(FIELD_ADVANCED)) { - // Set the advanced options - spiderParam.setMaxDepth(this.getIntValue(FIELD_MAX_DEPTH)); - spiderParam.setMaxDuration(this.getIntValue(FIELD_MAX_DURATION)); - spiderParam.setMaxChildren(this.getIntValue(FIELD_MAX_CHILDREN)); - spiderParam.setMaxParseSizeBytes(this.getIntValue(FIELD_MAX_PARSE_SIZE_BYTES)); - spiderParam.setSendRefererHeader(this.getBoolValue(FIELD_SEND_REFERER)); - spiderParam.setAcceptCookies(this.getBoolValue(FIELD_ACCEPT_COOKIES)); - spiderParam.setProcessForm(this.getBoolValue(FIELD_PROCESS_FORMS)); - spiderParam.setPostForm(this.getBoolValue(FIELD_POST_FORMS)); - spiderParam.setParseComments(this.getBoolValue(FIELD_PARSE_COMMENTS)); - spiderParam.setParseRobotsTxt(this.getBoolValue(FIELD_PARSE_ROBOTS)); - spiderParam.setParseSitemapXml(this.getBoolValue(FIELD_PARSE_SITEMAP)); - spiderParam.setParseSVNEntries(this.getBoolValue(FIELD_PARSE_SVN)); - spiderParam.setParseGit(this.getBoolValue(FIELD_PARSE_GIT)); - spiderParam.setHandleODataParametersVisited(this.getBoolValue(FIELD_HANDLE_ODATA)); - spiderParam.setThreadCount(extension.getSpiderParam().getThreadCount()); - spiderParam.setIrrelevantUrlParameters( - this.getStringValue(FIELD_IRRELEVANT_URL_PARAMETERS)); - - contextSpecificObjects.add(spiderParam); - } - - if (startUri != null) { - contextSpecificObjects.add(startUri); - - if (getBoolValue(FIELD_SUBTREE_ONLY)) { - contextSpecificObjects.add( - new org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter(startUri)); - } - } - - if (target == null || !this.getStringValue(FIELD_START).equals(getTargetText(target))) { - // Clear the target as it doesn't match the value entered manually - target = new Target((StructuralNode) null); - } - - // Save the adv option permanently for next time - extension.getSpiderParam().setShowAdvancedDialog(this.getBoolValue(FIELD_ADVANCED)); - - target.setRecurse(this.getBoolValue(FIELD_RECURSE)); - - if (target.getContext() == null && getSelectedContext() != null) { - target.setContext(getSelectedContext()); - } - - subtreeOnlyPreviousCheckedState = getBoolValue(FIELD_SUBTREE_ONLY); - - this.extension.startScan(target, getSelectedUser(), contextSpecificObjects.toArray()); - } - - @Override - public String validateFields() { - if (Control.Mode.safe == Control.getSingleton().getMode()) { - // The dialogue shouldn't be shown when in safe mode but if it is warn. - return Constant.messages.getString("spider.custom.notSafe.error"); - } - - if (this.isEmptyField(FIELD_START)) { - return Constant.messages.getString("spider.custom.nostart.error"); - } - - boolean noStartUri = true; - if (!getStringValue(FIELD_START).equals(getTargetText(target))) { - String url = this.getStringValue(FIELD_START); - try { - // Need both constructors as they catch slightly different issues ;) - new URI(url, true); - new URL(url); - } catch (Exception e) { - return Constant.messages.getString("spider.custom.nostart.error"); - } - - if (Control.getSingleton().getMode() == Control.Mode.protect) { - if (!extension.isTargetUriInScope(url)) { - return Constant.messages.getString("spider.custom.targetNotInScope.error", url); - } - } - noStartUri = false; - } - - if (this.target != null) { - if (!this.target.isValid()) { - return Constant.messages.getString("spider.custom.nostart.error"); - } - - if (Control.getSingleton().getMode() == Control.Mode.protect) { - String uri = extension.getTargetUriOutOfScope(target); - if (uri != null) { - return Constant.messages.getString("spider.custom.targetNotInScope.error", uri); - } - } - - List nodes = target.getStartNodes(); - if (nodes != null) { - for (StructuralNode node : nodes) { - if (node instanceof StructuralSiteNode) { - noStartUri = false; - break; - } - } - } - } - - if (getBoolValue(FIELD_SUBTREE_ONLY) && noStartUri) { - return Constant.messages.getString("spider.custom.noStartSubtreeOnly.error"); - } - - return null; - } - - /** - * Resets the spider dialogue to its default state. - * - * @since 2.5.0 - */ - void reset() { - target = null; - reset(true); - } - - @Override - public void cancelPressed() { - this.target = null; - super.cancelPressed(); - } -} diff --git a/zap/src/main/java/org/zaproxy/zap/extension/spider/SpiderEventPublisher.java b/zap/src/main/java/org/zaproxy/zap/extension/spider/SpiderEventPublisher.java deleted file mode 100644 index b455f502395..00000000000 --- a/zap/src/main/java/org/zaproxy/zap/extension/spider/SpiderEventPublisher.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2018 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.extension.spider; - -import org.zaproxy.zap.ZAP; -import org.zaproxy.zap.model.ScanEventPublisher; -import org.zaproxy.zap.model.Target; -import org.zaproxy.zap.users.User; - -/** - * @deprecated (2.12.0) See the spider add-on in zap-extensions instead. - */ -@Deprecated -public class SpiderEventPublisher extends ScanEventPublisher { - - private static SpiderEventPublisher publisher = null; - - @Override - public String getPublisherName() { - return SpiderEventPublisher.class.getCanonicalName(); - } - - public static synchronized SpiderEventPublisher getPublisher() { - if (publisher == null) { - publisher = new SpiderEventPublisher(); - ZAP.getEventBus().registerPublisher(publisher, getEvents()); - } - return publisher; - } - - public static void publishScanEvent(String event, int scanId) { - SpiderEventPublisher publisher = getPublisher(); - publisher.publishScanEvent(publisher, event, scanId); - } - - public static void publishScanEvent(String event, int scanId, Target target, User user) { - SpiderEventPublisher publisher = getPublisher(); - publisher.publishScanEvent(publisher, event, scanId, target, user); - } - - public static void publishScanProgressEvent(int scanId, int scanProgress) { - SpiderEventPublisher publisher = getPublisher(); - publisher.publishScanProgressEvent(publisher, scanId, scanProgress); - } -} diff --git a/zap/src/main/java/org/zaproxy/zap/extension/spider/SpiderMessagesTable.java b/zap/src/main/java/org/zaproxy/zap/extension/spider/SpiderMessagesTable.java deleted file mode 100644 index 1cc9c7e3888..00000000000 --- a/zap/src/main/java/org/zaproxy/zap/extension/spider/SpiderMessagesTable.java +++ /dev/null @@ -1,171 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2016 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.extension.spider; - -import java.awt.Component; -import javax.swing.Icon; -import javax.swing.ImageIcon; -import javax.swing.JLabel; -import javax.swing.SortOrder; -import javax.swing.table.TableModel; -import org.jdesktop.swingx.decorator.AbstractHighlighter; -import org.jdesktop.swingx.decorator.ComponentAdapter; -import org.jdesktop.swingx.renderer.DefaultTableRenderer; -import org.jdesktop.swingx.renderer.IconAware; -import org.jdesktop.swingx.renderer.IconValues; -import org.jdesktop.swingx.renderer.MappedValue; -import org.jdesktop.swingx.renderer.StringValues; -import org.parosproxy.paros.Constant; -import org.parosproxy.paros.control.Control; -import org.parosproxy.paros.extension.history.ExtensionHistory; -import org.parosproxy.paros.model.HistoryReference; -import org.zaproxy.zap.view.table.HistoryReferencesTable; - -@SuppressWarnings("serial") -/** - * @deprecated (2.12.0) See the spider add-on in zap-extensions instead. - */ -@Deprecated -class SpiderMessagesTable extends HistoryReferencesTable { - - private static final long serialVersionUID = -1910120966638329368L; - - private final ExtensionHistory extensionHistory; - - public SpiderMessagesTable(SpiderMessagesTableModel resultsModel) { - super(resultsModel); - - setName("SpiderMessagesTable"); - - setAutoCreateColumnsFromModel(false); - - getColumnExt(0) - .setCellRenderer( - new DefaultTableRenderer( - new MappedValue(StringValues.EMPTY, IconValues.NONE), - JLabel.CENTER)); - getColumnExt(0).setHighlighters(new ProcessedCellItemIconHighlighter(0)); - - getColumnExt(Constant.messages.getString("view.href.table.header.hrefid")) - .setVisible(false); - getColumnExt(Constant.messages.getString("view.href.table.header.timestamp.response")) - .setVisible(false); - getColumnExt(Constant.messages.getString("view.href.table.header.size.requestheader")) - .setVisible(false); - getColumnExt(Constant.messages.getString("view.href.table.header.size.requestbody")) - .setVisible(false); - - setSortOrder(1, SortOrder.ASCENDING); - - extensionHistory = - Control.getSingleton().getExtensionLoader().getExtension(ExtensionHistory.class); - } - - @Override - public void setModel(TableModel dataModel) { - // Keep the same column sorted when model is changed - int sortedcolumnIndex = getSortedColumnIndex(); - SortOrder sortOrder = getSortOrder(sortedcolumnIndex); - super.setModel(dataModel); - if (sortedcolumnIndex != -1) { - setSortOrder(sortedcolumnIndex, sortOrder); - } - } - - @Override - protected HistoryReference getHistoryReferenceAtViewRow(int row) { - HistoryReference historyReference = super.getHistoryReferenceAtViewRow(row); - if (historyReference == null) { - return null; - } - - if (extensionHistory == null - || extensionHistory.getHistoryReference(historyReference.getHistoryId()) == null) { - // Associated message was deleted in the meantime. - return null; - } - - return historyReference; - } - - /** - * A {@link org.jdesktop.swingx.decorator.Highlighter Highlighter} for a column that indicates, - * using icons and text, whether or not an entry was processed, that is, is or not in scope. - * - *

The expected type/class of the cell values is {@code ProcessedCellItem}. - */ - private static class ProcessedCellItemIconHighlighter extends AbstractHighlighter { - - /** The icon that indicates the entry was processed. */ - private static final ImageIcon PROCESSED_ICON = - new ImageIcon(SpiderMessagesTable.class.getResource("/resource/icon/16/152.png")); - - /** The icon that indicates the entry was not processed. */ - private static final ImageIcon NOT_PROCESSED_ICON = - new ImageIcon(SpiderMessagesTable.class.getResource("/resource/icon/16/149.png")); - - private final int columnIndex; - - public ProcessedCellItemIconHighlighter(final int columnIndex) { - this.columnIndex = columnIndex; - } - - @Override - protected Component doHighlight(Component component, ComponentAdapter adapter) { - org.zaproxy.zap.extension.spider.SpiderMessagesTableModel.ProcessedCellItem cell = - (org.zaproxy.zap.extension.spider.SpiderMessagesTableModel.ProcessedCellItem) - adapter.getValue(columnIndex); - - boolean processed = cell.isSuccessful(); - Icon icon = getProcessedIcon(processed); - if (component instanceof IconAware) { - ((IconAware) component).setIcon(icon); - } else if (component instanceof JLabel) { - ((JLabel) component).setIcon(icon); - } - - if (component instanceof JLabel) { - ((JLabel) component).setText(processed ? "" : cell.getLabel()); - } - - return component; - } - - private static Icon getProcessedIcon(final boolean processed) { - return processed ? PROCESSED_ICON : NOT_PROCESSED_ICON; - } - - /** - * {@inheritDoc} - * - *

Overridden to return true if the component is of type IconAware or of type JLabel, - * false otherwise. - * - *

Note: special casing JLabel is for backward compatibility - application highlighting - * code which doesn't use the Swingx renderers would stop working otherwise. - */ - // Method/JavaDoc copied from - // org.jdesktop.swingx.decorator.IconHighlighter#canHighlight(Component, ComponentAdapter) - @Override - protected boolean canHighlight(final Component component, final ComponentAdapter adapter) { - return component instanceof IconAware || component instanceof JLabel; - } - } -} diff --git a/zap/src/main/java/org/zaproxy/zap/extension/spider/SpiderMessagesTableModel.java b/zap/src/main/java/org/zaproxy/zap/extension/spider/SpiderMessagesTableModel.java deleted file mode 100644 index 2e9177d6fb8..00000000000 --- a/zap/src/main/java/org/zaproxy/zap/extension/spider/SpiderMessagesTableModel.java +++ /dev/null @@ -1,430 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2016 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.extension.spider; - -import java.awt.EventQueue; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import javax.swing.event.TableModelEvent; -import org.parosproxy.paros.Constant; -import org.parosproxy.paros.control.Control; -import org.parosproxy.paros.extension.history.ExtensionHistory; -import org.parosproxy.paros.model.HistoryReference; -import org.parosproxy.paros.model.HistoryReferenceEventPublisher; -import org.zaproxy.zap.ZAP; -import org.zaproxy.zap.eventBus.Event; -import org.zaproxy.zap.eventBus.EventConsumer; -import org.zaproxy.zap.extension.alert.AlertEventPublisher; -import org.zaproxy.zap.view.table.AbstractCustomColumnHistoryReferencesTableModel; -import org.zaproxy.zap.view.table.AbstractHistoryReferencesTableEntry; -import org.zaproxy.zap.view.table.DefaultHistoryReferencesTableEntry; - -@SuppressWarnings("serial") -/** - * @deprecated (2.12.0) See the spider add-on in zap-extensions instead. - */ -@Deprecated -class SpiderMessagesTableModel - extends AbstractCustomColumnHistoryReferencesTableModel< - SpiderMessagesTableModel.SpiderTableEntry> { - - private static final long serialVersionUID = 1093393768186896931L; - - private static final Column[] COLUMNS = - new Column[] { - Column.CUSTOM, - Column.HREF_ID, - Column.REQUEST_TIMESTAMP, - Column.RESPONSE_TIMESTAMP, - Column.METHOD, - Column.URL, - Column.STATUS_CODE, - Column.STATUS_REASON, - Column.RTT, - Column.SIZE_REQUEST_HEADER, - Column.SIZE_REQUEST_BODY, - Column.SIZE_RESPONSE_HEADER, - Column.SIZE_RESPONSE_BODY, - Column.HIGHEST_ALERT, - Column.TAGS - }; - - private static final String[] CUSTOM_COLUMN_NAMES = { - Constant.messages.getString("spider.table.messages.header.processed") - }; - - private static final ProcessedCellItem SUCCESSFULLY_PROCESSED_CELL_ITEM; - - private final ExtensionHistory extensionHistory; - private EventConsumerImpl eventConsumer; - - private List resources; - private Map idsToRows; - - private Map cacheProcessedCellItems; - - static { - SUCCESSFULLY_PROCESSED_CELL_ITEM = - new ProcessedCellItem( - true, - Constant.messages.getString( - "spider.table.messages.column.processed.successfully")); - } - - public SpiderMessagesTableModel() { - this(true); - } - - public SpiderMessagesTableModel(boolean createAlertEventConsumer) { - super(COLUMNS); - - resources = new ArrayList<>(); - idsToRows = new HashMap<>(); - cacheProcessedCellItems = new HashMap<>(); - - if (createAlertEventConsumer) { - eventConsumer = new EventConsumerImpl(); - extensionHistory = - Control.getSingleton() - .getExtensionLoader() - .getExtension(ExtensionHistory.class); - ZAP.getEventBus() - .registerConsumer( - eventConsumer, AlertEventPublisher.getPublisher().getPublisherName()); - ZAP.getEventBus() - .registerConsumer( - eventConsumer, - HistoryReferenceEventPublisher.getPublisher().getPublisherName()); - } else { - eventConsumer = null; - extensionHistory = null; - } - } - - @Override - public void addEntry(SpiderTableEntry entry) { - // Nothing to do, the entries are added with the following method. - } - - public void addHistoryReference( - HistoryReference historyReference, boolean processed, String reasonNotProcessed) { - HistoryReference latestHistoryReference = historyReference; - if (extensionHistory != null) { - latestHistoryReference = - extensionHistory.getHistoryReference(historyReference.getHistoryId()); - } - final SpiderTableEntry entry = - new SpiderTableEntry( - latestHistoryReference, - getProcessedCellItem(processed, reasonNotProcessed)); - EventQueue.invokeLater( - new Runnable() { - - @Override - public void run() { - final int row = resources.size(); - idsToRows.put(entry.getHistoryId(), row); - resources.add(entry); - fireTableRowsInserted(row, row); - } - }); - } - - private ProcessedCellItem getProcessedCellItem(boolean processed, String reasonNotProcessed) { - if (processed) { - return SUCCESSFULLY_PROCESSED_CELL_ITEM; - } - ProcessedCellItem processedCellItem = cacheProcessedCellItems.get(reasonNotProcessed); - if (processedCellItem == null) { - processedCellItem = new ProcessedCellItem(processed, reasonNotProcessed); - cacheProcessedCellItems.put(reasonNotProcessed, processedCellItem); - } - return processedCellItem; - } - - @Override - public void clear() { - cacheProcessedCellItems = new HashMap<>(); - resources = new ArrayList<>(); - idsToRows = new HashMap<>(); - fireTableDataChanged(); - - if (eventConsumer != null) { - ZAP.getEventBus() - .unregisterConsumer( - eventConsumer, AlertEventPublisher.getPublisher().getPublisherName()); - ZAP.getEventBus() - .unregisterConsumer( - eventConsumer, - HistoryReferenceEventPublisher.getPublisher().getPublisherName()); - eventConsumer = null; - } - } - - @Override - public void refreshEntryRow(int historyReferenceId) { - final DefaultHistoryReferencesTableEntry entry = getEntryWithHistoryId(historyReferenceId); - - if (entry != null) { - int rowIndex = getEntryRowIndex(historyReferenceId); - getEntryWithHistoryId(historyReferenceId).refreshCachedValues(); - - fireTableRowsUpdated(rowIndex, rowIndex); - } - } - - @Override - public void removeEntry(int historyReferenceId) { - // Nothing to do, the entries are not removed. - } - - @Override - public SpiderTableEntry getEntry(int rowIndex) { - return resources.get(rowIndex); - } - - @Override - public SpiderTableEntry getEntryWithHistoryId(int historyReferenceId) { - final int row = getEntryRowIndex(historyReferenceId); - if (row != -1) { - return resources.get(row); - } - return null; - } - - @Override - public int getEntryRowIndex(int historyReferenceId) { - final Integer row = idsToRows.get(historyReferenceId); - if (row != null) { - return row; - } - return -1; - } - - @Override - public int getRowCount() { - return resources.size(); - } - - @Override - protected Class getColumnClass(Column column) { - return AbstractHistoryReferencesTableEntry.getColumnClass(column); - } - - @Override - protected Object getPrototypeValue(Column column) { - return AbstractHistoryReferencesTableEntry.getPrototypeValue(column); - } - - @Override - public Object getValueAt(int rowIndex, int columnIndex) { - if (columnIndex == -1) { - return getEntry(rowIndex); - } - return super.getValueAt(rowIndex, columnIndex); - } - - @Override - protected Object getCustomValueAt(SpiderTableEntry entry, int columnIndex) { - if (getCustomColumnIndex(columnIndex) == 0) { - return entry.getProcessedCellItem(); - } - return null; - } - - @Override - protected String getCustomColumnName(int columnIndex) { - return CUSTOM_COLUMN_NAMES[getCustomColumnIndex(columnIndex)]; - } - - @Override - protected Class getCustomColumnClass(int columnIndex) { - if (getCustomColumnIndex(columnIndex) == 0) { - return ProcessedCellItem.class; - } - return null; - } - - @Override - protected Object getCustomPrototypeValue(int columnIndex) { - if (getCustomColumnIndex(columnIndex) == 0) { - return "Successful"; - } - return null; - } - - private void refreshEntryRows() { - if (resources.isEmpty()) { - return; - } - - for (SpiderTableEntry entry : resources) { - entry.refreshCachedValues(); - } - - fireTableChanged( - new TableModelEvent( - this, - 0, - resources.size() - 1, - getColumnIndex(Column.HIGHEST_ALERT), - TableModelEvent.UPDATE)); - } - - static class SpiderTableEntry extends DefaultHistoryReferencesTableEntry { - - private final ProcessedCellItem processedCellItem; - - public SpiderTableEntry( - HistoryReference historyReference, ProcessedCellItem processedCellItem) { - super(historyReference, COLUMNS); - this.processedCellItem = processedCellItem; - } - - public ProcessedCellItem getProcessedCellItem() { - return processedCellItem; - } - } - - static class ProcessedCellItem implements Comparable { - - private final boolean successful; - private final String label; - - public ProcessedCellItem(boolean successful, String label) { - this.successful = successful; - this.label = label; - } - - public boolean isSuccessful() { - return successful; - } - - public String getLabel() { - return label; - } - - @Override - public String toString() { - return label; - } - - @Override - public int hashCode() { - return 31 * (successful ? 1231 : 1237); - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null) { - return false; - } - if (getClass() != obj.getClass()) { - return false; - } - ProcessedCellItem other = (ProcessedCellItem) obj; - if (successful != other.successful) { - return false; - } - return true; - } - - @Override - public int compareTo(ProcessedCellItem other) { - if (other == null) { - return 1; - } - if (successful && !other.successful) { - return 1; - } else if (!successful && other.successful) { - return -1; - } - return label.compareTo(other.label); - } - } - - private class EventConsumerImpl implements EventConsumer { - - @Override - public void eventReceived(Event event) { - switch (event.getEventType()) { - case HistoryReferenceEventPublisher.EVENT_TAG_ADDED: - case HistoryReferenceEventPublisher.EVENT_TAG_REMOVED: - case HistoryReferenceEventPublisher.EVENT_TAGS_SET: - refreshEntry( - Integer.valueOf( - event.getParameters() - .get( - HistoryReferenceEventPublisher - .FIELD_HISTORY_REFERENCE_ID))); - break; - case AlertEventPublisher.ALERT_ADDED_EVENT: - case AlertEventPublisher.ALERT_CHANGED_EVENT: - case AlertEventPublisher.ALERT_REMOVED_EVENT: - refreshEntry( - Integer.valueOf( - event.getParameters() - .get(AlertEventPublisher.HISTORY_REFERENCE_ID))); - break; - case AlertEventPublisher.ALL_ALERTS_REMOVED_EVENT: - refreshEntries(); - break; - default: - } - } - - private void refreshEntry(final int id) { - if (EventQueue.isDispatchThread()) { - refreshEntryRow(id); - return; - } - - EventQueue.invokeLater( - new Runnable() { - - @Override - public void run() { - refreshEntry(id); - } - }); - } - - private void refreshEntries() { - if (EventQueue.isDispatchThread()) { - refreshEntryRows(); - return; - } - - EventQueue.invokeLater( - new Runnable() { - - @Override - public void run() { - refreshEntries(); - } - }); - } - } -} diff --git a/zap/src/main/java/org/zaproxy/zap/extension/spider/SpiderPanel.java b/zap/src/main/java/org/zaproxy/zap/extension/spider/SpiderPanel.java deleted file mode 100644 index a15ba4d45a0..00000000000 --- a/zap/src/main/java/org/zaproxy/zap/extension/spider/SpiderPanel.java +++ /dev/null @@ -1,585 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2010 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.extension.spider; - -import java.awt.BorderLayout; -import java.awt.Component; -import java.awt.EventQueue; -import java.awt.event.ActionEvent; -import java.awt.event.ActionListener; -import java.awt.event.KeyEvent; -import java.lang.reflect.InvocationTargetException; -import javax.swing.Icon; -import javax.swing.ImageIcon; -import javax.swing.JButton; -import javax.swing.JLabel; -import javax.swing.JPanel; -import javax.swing.JPopupMenu; -import javax.swing.JScrollPane; -import javax.swing.JTabbedPane; -import javax.swing.JToolBar; -import javax.swing.ListSelectionModel; -import javax.swing.event.ChangeEvent; -import javax.swing.event.ChangeListener; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.jdesktop.swingx.JXTable; -import org.jdesktop.swingx.renderer.DefaultTableRenderer; -import org.jdesktop.swingx.renderer.IconValues; -import org.jdesktop.swingx.renderer.MappedValue; -import org.jdesktop.swingx.renderer.StringValues; -import org.parosproxy.paros.Constant; -import org.parosproxy.paros.model.SiteNode; -import org.parosproxy.paros.view.View; -import org.zaproxy.zap.model.ScanController; -import org.zaproxy.zap.model.ScanListenner2; -import org.zaproxy.zap.utils.DisplayUtils; -import org.zaproxy.zap.utils.TableExportButton; -import org.zaproxy.zap.view.ScanPanel2; -import org.zaproxy.zap.view.ZapTable; -import org.zaproxy.zap.view.table.decorator.AbstractTableCellItemIconHighlighter; - -/** - * The Class SpiderPanel implements the Panel that is shown to the users when selecting the Spider - * Scan Tab. - * - * @deprecated (2.12.0) See the spider add-on in zap-extensions instead. - */ -@SuppressWarnings("serial") -@Deprecated -public class SpiderPanel extends ScanPanel2> - implements ScanListenner2 { - - /** - * The name of the spider's HTTP messages container. - * - * @since 2.5.0 - * @see org.zaproxy.zap.view.messagecontainer.http.HttpMessageContainer - */ - public static final String HTTP_MESSAGE_CONTAINER_NAME = "SpiderHttpMessageContainer"; - - /** The Constant serialVersionUID. */ - private static final long serialVersionUID = 1L; - - /** The Constant log. */ - private static final Logger log = LogManager.getLogger(SpiderPanel.class); - - private static final String ZERO_REQUESTS_LABEL_TEXT = "0"; - - private static final SpiderPanelTableModel EMPTY_URLS_TABLE_MODEL = new SpiderPanelTableModel(); - - private static final SpiderPanelTableModel EMPTY_URLS_NO_FLAGS_TABLE_MODEL = - new SpiderPanelTableModel(false); - - private static final SpiderMessagesTableModel EMPTY_MESSAGES_TABLE_MODEL = - new SpiderMessagesTableModel(false); - - /** The Constant defining the PANEL's NAME. */ - public static final String PANEL_NAME = "SpiderPanel"; - - private static final String ADDED_NODES_CONTAINER_NAME = "SpiderAddedNodesContainer"; - - /** - * The main panel, where the {@link #tabbedPane} or {@link #urlsTableScrollPane} are added. - * - *

Lazily initialised. - * - * @see #getWorkPanel() - */ - private JPanel mainPanel; - - /** The {@code JTabbedPane} used to show the tabs for URLs found and HTTP messages sent. */ - private JTabbedPane tabbedPane; - - private JButton scanButton = null; - - /** - * The table with URLs found. - * - *

Lazily initialised. - * - * @see #getUrlsTable() - * @see #urlsTableScrollPane - */ - private ZapTable urlsTable; - - /** - * The table with added nodes. - * - *

Lazily initialised. - * - * @see #getAddedNodesTable() - * @see #addedNodesTableScrollPane - */ - private ZapTable addedNodesTable; - - /** - * The scroll pane for the URLs table. - * - *

Lazily initialised. - * - * @see #getUrlsTableScrollPane() - * @see #urlsTable - */ - private JScrollPane urlsTableScrollPane; - - /** - * The scroll pane for the added nodes table. - * - *

Lazily initialised. - * - * @see #getAddedNodesTableScrollPane() - * @see #addedNodesTable - */ - private JScrollPane addedNodesTableScrollPane; - - /** - * The table with HTTP messages sent. - * - *

Lazily initialised. - * - * @see #getMessagesTable() - * @see #messagesTableScrollPane - */ - private SpiderMessagesTable messagesTable; - - /** - * The scroll pane for the HTTP messages table. - * - *

Lazily initialised. - * - * @see #getMessagesTableScrollPanel() - * @see #messagesTable - */ - private JScrollPane messagesTableScrollPane; - - /** The found count name label. */ - private JLabel foundCountNameLabel; - - /** The found count value label. */ - private JLabel foundCountValueLabel; - - /** The added count name label. */ - private JLabel addedCountNameLabel; - - /** The added count value label. */ - private JLabel addedCountValueLabel; - - private TableExportButton exportButton; - - private ExtensionSpider extension = null; - - /** - * Instantiates a new spider panel. - * - * @param extension the extension - * @param spiderScanParam the spider scan parameters - */ - public SpiderPanel( - ExtensionSpider extension, org.zaproxy.zap.spider.SpiderParam spiderScanParam) { - super( - "spider", - new ImageIcon(SpiderPanel.class.getResource("/resource/icon/16/spider.png")), - extension); - - tabbedPane.addChangeListener( - new ChangeListener() { - @Override - public void stateChanged(ChangeEvent e) { - switch (tabbedPane.getSelectedIndex()) { - case 0: - getExportButton().setTable(getUrlsTable()); - break; - case 1: - getExportButton().setTable(getAddedNodesTable()); - break; - case 2: - getExportButton().setTable(getMessagesTable()); - break; - } - } - }); - - this.extension = extension; - this.setDefaultAccelerator( - extension - .getView() - .getMenuShortcutKeyStroke(KeyEvent.VK_D, KeyEvent.SHIFT_DOWN_MASK, false)); - this.setMnemonic(Constant.messages.getChar("spider.panel.mnemonic")); - } - - /** - * This method initializes the working Panel. - * - * @return javax.swing.JScrollPane - */ - @Override - protected JPanel getWorkPanel() { - if (mainPanel == null) { - mainPanel = new JPanel(new BorderLayout()); - - tabbedPane = new JTabbedPane(); - tabbedPane.addTab( - Constant.messages.getString("spider.panel.tab.urls"), getUrlsTableScrollPane()); - tabbedPane.addTab( - Constant.messages.getString("spider.panel.tab.addednodes"), - getAddedNodesTableScrollPane()); - tabbedPane.addTab( - Constant.messages.getString("spider.panel.tab.messages"), - getMessagesTableScrollPanel()); - tabbedPane.setSelectedIndex(0); - - mainPanel.add(tabbedPane); - } - return mainPanel; - } - - private JScrollPane getUrlsTableScrollPane() { - if (urlsTableScrollPane == null) { - urlsTableScrollPane = new JScrollPane(); - urlsTableScrollPane.setName("SpiderUrlsPane"); - urlsTableScrollPane.setViewportView(getUrlsTable()); - } - return urlsTableScrollPane; - } - - private JScrollPane getAddedNodesTableScrollPane() { - if (addedNodesTableScrollPane == null) { - addedNodesTableScrollPane = new JScrollPane(); - addedNodesTableScrollPane.setName("SpiderAddedUrlsPane"); - addedNodesTableScrollPane.setViewportView(getAddedNodesTable()); - } - return addedNodesTableScrollPane; - } - - /** - * Gets the scan results table. - * - * @return the scan results table - */ - private JXTable getUrlsTable() { - if (urlsTable == null) { - // Create the table with a default, empty TableModel and the proper settings - urlsTable = new ZapTable(EMPTY_URLS_TABLE_MODEL); - urlsTable.setColumnSelectionAllowed(false); - urlsTable.setCellSelectionEnabled(false); - urlsTable.setRowSelectionAllowed(true); - urlsTable.setAutoCreateRowSorter(true); - - urlsTable.setAutoCreateColumnsFromModel(false); - urlsTable - .getColumnExt(0) - .setCellRenderer( - new DefaultTableRenderer( - new MappedValue(StringValues.EMPTY, IconValues.NONE), - JLabel.CENTER)); - urlsTable.getColumnExt(0).setHighlighters(new ProcessedCellItemIconHighlighter(0)); - - urlsTable.getColumnModel().getColumn(0).setMinWidth(80); - urlsTable.getColumnModel().getColumn(0).setPreferredWidth(90); // processed - - urlsTable.getColumnModel().getColumn(1).setMinWidth(60); - urlsTable.getColumnModel().getColumn(1).setPreferredWidth(70); // method - - urlsTable.getColumnModel().getColumn(2).setMinWidth(300); // name - - urlsTable.getColumnModel().getColumn(3).setMinWidth(50); - urlsTable.getColumnModel().getColumn(3).setPreferredWidth(250); // flags - - urlsTable.setName(PANEL_NAME); - urlsTable.setDoubleBuffered(true); - urlsTable.setSelectionMode(ListSelectionModel.MULTIPLE_INTERVAL_SELECTION); - urlsTable.setComponentPopupMenu( - new JPopupMenu() { - - private static final long serialVersionUID = 6608291059686282641L; - - @Override - public void show(Component invoker, int x, int y) { - View.getSingleton().getPopupMenu().show(invoker, x, y); - } - }); - } - return urlsTable; - } - - private JXTable getAddedNodesTable() { - if (addedNodesTable == null) { - // Create the table with a default, empty TableModel and the proper settings - addedNodesTable = new ZapTable(EMPTY_URLS_NO_FLAGS_TABLE_MODEL); - addedNodesTable.setColumnSelectionAllowed(false); - addedNodesTable.setCellSelectionEnabled(false); - addedNodesTable.setRowSelectionAllowed(true); - addedNodesTable.setAutoCreateRowSorter(true); - - addedNodesTable.setAutoCreateColumnsFromModel(false); - addedNodesTable - .getColumnExt(0) - .setCellRenderer( - new DefaultTableRenderer( - new MappedValue(StringValues.EMPTY, IconValues.NONE), - JLabel.CENTER)); - addedNodesTable - .getColumnExt(0) - .setHighlighters(new ProcessedCellItemIconHighlighter(0)); - - addedNodesTable.getColumnModel().getColumn(0).setMinWidth(80); - addedNodesTable.getColumnModel().getColumn(0).setPreferredWidth(90); // processed - - addedNodesTable.getColumnModel().getColumn(1).setMinWidth(60); - addedNodesTable.getColumnModel().getColumn(1).setPreferredWidth(70); // method - - addedNodesTable.getColumnModel().getColumn(2).setMinWidth(400); // name - addedNodesTable.getColumnModel().getColumn(2).setPreferredWidth(1000); - - addedNodesTable.setName(ADDED_NODES_CONTAINER_NAME); - addedNodesTable.setDoubleBuffered(true); - addedNodesTable.setSelectionMode(ListSelectionModel.MULTIPLE_INTERVAL_SELECTION); - addedNodesTable.setComponentPopupMenu( - new JPopupMenu() { - - private static final long serialVersionUID = 6608291059686282641L; - - @Override - public void show(Component invoker, int x, int y) { - View.getSingleton().getPopupMenu().show(invoker, x, y); - } - }); - } - return addedNodesTable; - } - - private JScrollPane getMessagesTableScrollPanel() { - if (messagesTableScrollPane == null) { - messagesTableScrollPane = new JScrollPane(); - messagesTableScrollPane.setName("SpiderMessagesPane"); - messagesTableScrollPane.setViewportView(getMessagesTable()); - } - return messagesTableScrollPane; - } - - private SpiderMessagesTable getMessagesTable() { - if (messagesTable == null) { - messagesTable = new SpiderMessagesTable(EMPTY_MESSAGES_TABLE_MODEL); - messagesTable.setName(HTTP_MESSAGE_CONTAINER_NAME); - } - return messagesTable; - } - - /** - * Gets the label storing the name of the count of found URIs. - * - * @return the found count name label - */ - private JLabel getFoundCountNameLabel() { - if (foundCountNameLabel == null) { - foundCountNameLabel = new JLabel(); - foundCountNameLabel.setText(Constant.messages.getString("spider.toolbar.found.label")); - } - return foundCountNameLabel; - } - - /** - * Gets the label storing the value for count of found URIs. - * - * @return the found count value label - */ - private JLabel getFoundCountValueLabel() { - if (foundCountValueLabel == null) { - foundCountValueLabel = new JLabel(); - foundCountValueLabel.setText(ZERO_REQUESTS_LABEL_TEXT); - } - return foundCountValueLabel; - } - - /** - * Gets the label storing the name of the count of added URIs. - * - * @return the found count name label - */ - private JLabel getAddedCountNameLabel() { - if (addedCountNameLabel == null) { - addedCountNameLabel = new JLabel(); - addedCountNameLabel.setText(Constant.messages.getString("spider.toolbar.added.label")); - } - return addedCountNameLabel; - } - - /** - * Gets the label storing the value for count of added URIs. - * - * @return the added count value label - */ - private JLabel getAddedCountValueLabel() { - if (addedCountValueLabel == null) { - addedCountValueLabel = new JLabel(); - addedCountValueLabel.setText(ZERO_REQUESTS_LABEL_TEXT); - } - return addedCountValueLabel; - } - - @Override - protected int addToolBarElements(JToolBar toolBar, Location location, int gridX) { - if (ScanPanel2.Location.afterProgressBar == location) { - toolBar.add(new JToolBar.Separator(), getGBC(gridX++, 0)); - toolBar.add(getFoundCountNameLabel(), getGBC(gridX++, 0)); - toolBar.add(getFoundCountValueLabel(), getGBC(gridX++, 0)); - - toolBar.add(new JToolBar.Separator(), getGBC(gridX++, 0)); - toolBar.add(getAddedCountNameLabel(), getGBC(gridX++, 0)); - toolBar.add(getAddedCountValueLabel(), getGBC(gridX++, 0)); - - toolBar.add(new JToolBar.Separator(), getGBC(gridX++, 0)); - toolBar.add(getExportButton(), getGBC(gridX++, 0)); - } - return gridX; - } - - private TableExportButton getExportButton() { - if (exportButton == null) { - exportButton = new TableExportButton<>(getUrlsTable()); - } - return exportButton; - } - - /** Update the count of found URIs. */ - protected void updateFoundCount() { - SpiderScan sc = this.getSelectedScanner(); - if (sc != null) { - this.getFoundCountValueLabel().setText(Integer.toString(sc.getNumberOfURIsFound())); - } else { - this.getFoundCountValueLabel().setText(ZERO_REQUESTS_LABEL_TEXT); - } - } - - /** Update the count of added nodes. */ - protected void updateAddedCount() { - SpiderScan sc = this.getSelectedScanner(); - if (sc != null) { - this.getAddedCountValueLabel().setText(Integer.toString(sc.getNumberOfNodesAdded())); - } else { - this.getAddedCountValueLabel().setText(ZERO_REQUESTS_LABEL_TEXT); - } - } - - @Override - protected void switchView(final SpiderScan scanner) { - if (View.isInitialised() && !EventQueue.isDispatchThread()) { - try { - EventQueue.invokeAndWait( - new Runnable() { - - @Override - public void run() { - switchView(scanner); - } - }); - } catch (InvocationTargetException | InterruptedException e) { - log.error("Failed to switch view: {}", e.getMessage(), e); - } - return; - } - if (scanner != null) { - getUrlsTable().setModel(scanner.getResultsTableModel()); - getMessagesTable().setModel(scanner.getMessagesTableModel()); - getAddedNodesTable().setModel(scanner.getAddedNodesTableModel()); - } else { - getUrlsTable().setModel(EMPTY_URLS_TABLE_MODEL); - getMessagesTable().setModel(EMPTY_MESSAGES_TABLE_MODEL); - getAddedNodesTable().setModel(EMPTY_URLS_TABLE_MODEL); - } - this.updateFoundCount(); - this.updateAddedCount(); - } - - @Override - public JButton getNewScanButton() { - if (scanButton == null) { - scanButton = new JButton(Constant.messages.getString("spider.toolbar.button.new")); - scanButton.setIcon( - DisplayUtils.getScaledIcon( - new ImageIcon( - SpiderPanel.class.getResource( - "/resource/icon/16/spider.png")))); - scanButton.addActionListener( - new ActionListener() { - @Override - public void actionPerformed(ActionEvent e) { - extension.showSpiderDialog(getSiteTreeTarget()); - } - }); - } - return scanButton; - } - - @Override - protected int getNumberOfScansToShow() { - return extension.getSpiderParam().getMaxScansInUI(); - } - - /** - * A {@link org.jdesktop.swingx.decorator.Highlighter Highlighter} for a column that indicates, - * using icons, whether or not an entry was processed, that is, is or not in scope. - * - *

The expected type/class of the cell values is {@code Boolean}. - */ - private static class ProcessedCellItemIconHighlighter - extends AbstractTableCellItemIconHighlighter { - - /** The icon that indicates the entry was processed. */ - private static final ImageIcon PROCESSED_ICON = - new ImageIcon(SpiderPanelTableModel.class.getResource("/resource/icon/16/152.png")); - - /** The icon that indicates the entry was not processed. */ - private static final ImageIcon NOT_PROCESSED_ICON = - new ImageIcon(SpiderPanelTableModel.class.getResource("/resource/icon/16/149.png")); - - public ProcessedCellItemIconHighlighter(final int columnIndex) { - super(columnIndex); - } - - @Override - protected Icon getIcon(final Object cellItem) { - return getProcessedIcon((Boolean) cellItem); - } - - private static Icon getProcessedIcon(final boolean processed) { - return processed ? PROCESSED_ICON : NOT_PROCESSED_ICON; - } - - @Override - protected boolean isHighlighted(final Object cellItem) { - return true; - } - } - - private SiteNode getSiteTreeTarget() { - if (!extension.getView().getSiteTreePanel().getTreeSite().isSelectionEmpty()) { - return (SiteNode) - extension - .getView() - .getSiteTreePanel() - .getTreeSite() - .getSelectionPath() - .getLastPathComponent(); - } - return null; - } -} diff --git a/zap/src/main/java/org/zaproxy/zap/extension/spider/SpiderPanelTableModel.java b/zap/src/main/java/org/zaproxy/zap/extension/spider/SpiderPanelTableModel.java deleted file mode 100644 index 622367387df..00000000000 --- a/zap/src/main/java/org/zaproxy/zap/extension/spider/SpiderPanelTableModel.java +++ /dev/null @@ -1,242 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2012 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.extension.spider; - -import java.util.ArrayList; -import java.util.List; -import javax.swing.table.AbstractTableModel; -import org.parosproxy.paros.Constant; - -/** The Class HttpSessionsTableModel that is used as a TableModel for the Http Sessions Panel. */ -@SuppressWarnings("serial") -/** - * The Class HttpSessionsTableModel that is used as a TableModel for the Http Sessions Panel. - * - * @deprecated (2.12.0) See the spider add-on in zap-extensions instead. - */ -@Deprecated -public class SpiderPanelTableModel extends AbstractTableModel { - - /** The Constant serialVersionUID. */ - private static final long serialVersionUID = -6380136823410869457L; - - /** The column names. */ - private static final String[] COLUMN_NAMES = { - Constant.messages.getString("spider.table.header.inScope"), - Constant.messages.getString("spider.table.header.method"), - Constant.messages.getString("spider.table.header.uri"), - Constant.messages.getString("spider.table.header.flags") - }; - - /** The Constant defining the COLUMN COUNT. */ - private static final int COLUMN_COUNT = COLUMN_NAMES.length; - - /** The Spider scan results. */ - private List scanResults; - - private boolean incFlags; - - /** Instantiates a new spider panel table model. */ - public SpiderPanelTableModel() { - this(true); - - scanResults = new ArrayList<>(); - } - - /** Instantiates a new spider panel table model. */ - public SpiderPanelTableModel(boolean incFlags) { - super(); - this.incFlags = incFlags; - - scanResults = new ArrayList<>(); - } - - @Override - public String getColumnName(int column) { - return COLUMN_NAMES[column]; - } - - @Override - public int getColumnCount() { - if (incFlags) { - return COLUMN_COUNT; - } else { - return COLUMN_COUNT - 1; - } - } - - @Override - public int getRowCount() { - return scanResults.size(); - } - - @Override - public Object getValueAt(int row, int col) { - // Get the ScanResult and the required field - SpiderScanResult result = scanResults.get(row); - switch (col) { - case 0: - return result.processed; - case 1: - return result.method; - case 2: - return result.uri; - case 3: - return result.flags; - default: - return null; - } - } - - /** Removes all the elements. Method is synchronized internally. */ - public void removeAllElements() { - scanResults.clear(); - fireTableDataChanged(); - } - - /** - * Adds a new spider scan result. Method is synchronized internally. - * - * @param uri the uri - * @param method the method - * @param flags the flags - * @param skipped {@code true} if the result was skipped, {@code false} otherwise - */ - public void addScanResult(String uri, String method, String flags, boolean skipped) { - SpiderScanResult result = new SpiderScanResult(uri, method, flags, !skipped); - scanResults.add(result); - fireTableRowsInserted(scanResults.size() - 1, scanResults.size() - 1); - } - - /** - * Removes the scan result for a particular uri and method. Method is synchronized internally. - * - * @param uri the uri - * @param method the method - */ - public void removesScanResult(String uri, String method) { - SpiderScanResult toRemove = new SpiderScanResult(uri, method); - int index = scanResults.indexOf(toRemove); - if (index >= 0) { - scanResults.remove(index); - fireTableRowsDeleted(index, index); - } - } - - /** - * Returns the type of column for given column index. - * - * @param columnIndex the column index - * @return the column class - */ - @Override - public Class getColumnClass(int columnIndex) { - switch (columnIndex) { - case 0: - return Boolean.class; - case 1: - return String.class; - case 2: - return String.class; - case 3: - return String.class; - } - return null; - } - - /** - * The Class SpiderScanResult that stores an entry in the table (a result for the spidering - * process). - */ - private static class SpiderScanResult { - - /** The uri. */ - protected String uri; - - /** The method. */ - protected String method; - - /** The flags. */ - protected String flags; - - /** The in scope. */ - protected boolean processed; - - /** - * Instantiates a new spider scan result. - * - * @param uri the uri - * @param method the method - */ - protected SpiderScanResult(String uri, String method) { - super(); - this.uri = uri; - this.method = method; - } - - /** - * Instantiates a new spider scan result. - * - * @param uri the uri - * @param method the method - * @param flags the flags - * @param processed {@code true} if the result was processed, {@code false} otherwise - */ - protected SpiderScanResult(String uri, String method, String flags, boolean processed) { - super(); - this.uri = uri; - this.method = method; - this.flags = flags; - this.processed = processed; - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((method == null) ? 0 : method.hashCode()); - result = prime * result + ((uri == null) ? 0 : uri.hashCode()); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) return true; - if (!(obj instanceof SpiderScanResult)) return false; - // Removed some irrelevant checks, to speed up the method. - SpiderScanResult other = (SpiderScanResult) obj; - if (method == null) { - if (other.method != null) return false; - } else if (!method.equals(other.method)) return false; - if (uri == null) { - if (other.uri != null) return false; - } else if (!uri.equals(other.uri)) return false; - return true; - } - } - - public List getAddedNodes() { - List list = new ArrayList<>(this.scanResults.size()); - for (SpiderScanResult res : this.scanResults) { - list.add(res.uri); - } - return list; - } -} diff --git a/zap/src/main/java/org/zaproxy/zap/extension/spider/SpiderResource.java b/zap/src/main/java/org/zaproxy/zap/extension/spider/SpiderResource.java deleted file mode 100644 index 0139e6c9938..00000000000 --- a/zap/src/main/java/org/zaproxy/zap/extension/spider/SpiderResource.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2014 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.extension.spider; - -/** - * A resource (e.g. webpage) found while spidering. - * - *

Contains the HTTP method used to fetch the resource, status code and reason, URI and the ID of - * the corresponding (persisted) HTTP message. - * - * @deprecated (2.12.0) See the spider add-on in zap-extensions instead. - */ -@Deprecated -public class SpiderResource { - - private final int historyId; - private final String method; - private final String uri; - private final int statusCode; - private final String statusReason; - private final boolean processed; - private final String reasonNotProcessed; - - public SpiderResource( - int historyId, - String method, - String uri, - int statusCode, - String statusReason, - boolean processed, - String reasonNotProcessed) { - this.historyId = historyId; - this.method = method; - this.uri = uri; - this.statusCode = statusCode; - this.statusReason = statusReason; - this.processed = processed; - this.reasonNotProcessed = reasonNotProcessed == null ? "" : reasonNotProcessed; - } - - public int getHistoryId() { - return historyId; - } - - public String getMethod() { - return method; - } - - public String getUri() { - return uri; - } - - public int getStatusCode() { - return statusCode; - } - - public String getStatusReason() { - return statusReason; - } - - public boolean isProcessed() { - return processed; - } - - public String getReasonNotProcessed() { - return reasonNotProcessed; - } -} diff --git a/zap/src/main/java/org/zaproxy/zap/extension/spider/SpiderScan.java b/zap/src/main/java/org/zaproxy/zap/extension/spider/SpiderScan.java deleted file mode 100644 index e7a4f1749ea..00000000000 --- a/zap/src/main/java/org/zaproxy/zap/extension/spider/SpiderScan.java +++ /dev/null @@ -1,557 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2014 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.extension.spider; - -import java.awt.EventQueue; -import java.util.ArrayList; -import java.util.Collections; -import java.util.ConcurrentModificationException; -import java.util.EnumSet; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; -import javax.swing.table.TableModel; -import org.apache.commons.httpclient.URI; -import org.parosproxy.paros.network.HttpMessage; -import org.parosproxy.paros.network.HttpRequestHeader; -import org.parosproxy.paros.network.HttpResponseHeader; -import org.parosproxy.paros.view.View; -import org.zaproxy.zap.model.GenericScanner2; -import org.zaproxy.zap.model.ScanEventPublisher; -import org.zaproxy.zap.model.ScanListenner; -import org.zaproxy.zap.model.ScanListenner2; -import org.zaproxy.zap.model.Target; -import org.zaproxy.zap.users.User; -import org.zaproxy.zap.utils.Stats; - -/** - * @deprecated (2.12.0) See the spider add-on in zap-extensions instead. - */ -@Deprecated -public class SpiderScan - implements ScanListenner, org.zaproxy.zap.spider.SpiderListener, GenericScanner2 { - - public static final String SPIDER_SCAN_STARTED_STATS = "stats.spider.started"; - public static final String SPIDER_SCAN_STOPPED_STATS = "stats.spider.stopped"; - public static final String SPIDER_SCAN_TIME_STATS = "stats.spider.time"; - public static final String SPIDER_URL_FOUND_STATS = "stats.spider.url.found"; - public static final String SPIDER_URL_ERROR_STATS = "stats.spider.url.error"; - - private static enum State { - NOT_STARTED, - RUNNING, - PAUSED, - FINISHED - } - - private static final EnumSet - FETCH_STATUS_IN_SCOPE = - EnumSet.of( - org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus.VALID, - org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus.SEED); - - private static final EnumSet - FETCH_STATUS_OUT_OF_SCOPE = - EnumSet.of( - org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus.OUT_OF_SCOPE, - org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus.OUT_OF_CONTEXT, - org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus.USER_RULES); - - private final Lock lock; - - private int scanId; - private Target target; - private User user; - - private String displayName = ""; - - /** - * Counter for number of URIs, in and out of scope, found during the scan. - * - *

The counter is incremented when a new URI is found. - * - * @see #foundURI(String, String, org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus) - * @see #getNumberOfURIsFound() - */ - private AtomicInteger numberOfURIsFound; - - private Set foundURIs; - - private List resourcesFound; - - private List resourcesIoErrors; - - private Set foundURIsOutOfScope; - - private SpiderThread spiderThread = null; - - private State state; - - private int progress; - - private ScanListenner2 listener = null; - - private volatile boolean cleared; - - /** - * The table model of the messages sent. - * - *

Lazily initialised. - * - * @see #getMessagesTableModel() - * @see #addMessageToMessagesTableModel(org.zaproxy.zap.spider.SpiderTaskResult) - */ - private SpiderMessagesTableModel messagesTableModel; - - /** - * Constructs a {@code SpiderScan} with the given data. - * - * @param extension the extension to obtain configurations and notify the view - * @param spiderParams the spider options - * @param target the spider target - * @param spiderURI the starting URI, may be {@code null}. - * @param scanUser the user to be used in the scan, may be {@code null}. - * @param scanId the ID of the scan - * @param name the name that identifies the target - * @since 2.6.0 - */ - public SpiderScan( - ExtensionSpider extension, - org.zaproxy.zap.spider.SpiderParam spiderParams, - Target target, - URI spiderURI, - User scanUser, - int scanId, - String name) { - lock = new ReentrantLock(); - this.scanId = scanId; - this.target = target; - this.user = scanUser; - setDisplayName(name); - - numberOfURIsFound = new AtomicInteger(); - foundURIs = Collections.synchronizedSet(new HashSet<>()); - resourcesFound = Collections.synchronizedList(new ArrayList<>()); - resourcesIoErrors = Collections.synchronizedList(new ArrayList<>()); - foundURIsOutOfScope = Collections.synchronizedSet(new HashSet<>()); - - state = State.NOT_STARTED; - - spiderThread = - new SpiderThread(Integer.toString(scanId), extension, spiderParams, name, this); - - spiderThread.setStartURI(spiderURI); - spiderThread.setStartNode(target.getStartNode()); - spiderThread.setScanContext(target.getContext()); - spiderThread.setScanAsUser(scanUser); - spiderThread.setJustScanInScope(target.isInScopeOnly()); - spiderThread.setScanChildren(target.isRecurse()); - } - - /** - * Returns the ID of the scan. - * - * @return the ID of the scan - */ - @Override - public int getScanId() { - return scanId; - } - - /** - * Returns the {@code String} representation of the scan state (not started, running, paused or - * finished). - * - * @return the {@code String} representation of the scan state. - */ - public String getState() { - lock.lock(); - try { - return state.toString(); - } finally { - lock.unlock(); - } - } - - /** - * Returns the progress of the scan, an integer between 0 and 100. - * - * @return the progress of the scan. - */ - @Override - public int getProgress() { - return progress; - } - - /** - * Starts the scan. - * - *

The call to this method has no effect if the scan was already started. - */ - public void start() { - lock.lock(); - try { - if (State.NOT_STARTED.equals(state)) { - spiderThread.addSpiderListener(this); - spiderThread.start(); - state = State.RUNNING; - SpiderEventPublisher.publishScanEvent( - ScanEventPublisher.SCAN_STARTED_EVENT, this.scanId, this.target, user); - Stats.incCounter(SPIDER_SCAN_STARTED_STATS); - } - } finally { - lock.unlock(); - } - } - - /** - * Pauses the scan. - * - *

The call to this method has no effect if the scan is not running. - */ - @Override - public void pauseScan() { - lock.lock(); - try { - if (State.RUNNING.equals(state)) { - spiderThread.pauseScan(); - state = State.PAUSED; - SpiderEventPublisher.publishScanEvent( - ScanEventPublisher.SCAN_PAUSED_EVENT, this.scanId); - } - } finally { - lock.unlock(); - } - } - - /** - * Resumes the scan. - * - *

The call to this method has no effect if the scan is not paused. - */ - @Override - public void resumeScan() { - lock.lock(); - try { - if (State.PAUSED.equals(state)) { - spiderThread.resumeScan(); - state = State.RUNNING; - SpiderEventPublisher.publishScanEvent( - ScanEventPublisher.SCAN_RESUMED_EVENT, this.scanId); - } - } finally { - lock.unlock(); - } - } - - /** - * Stops the scan. - * - *

The call to this method has no effect if the scan was not yet started or has already - * finished. - */ - @Override - public void stopScan() { - lock.lock(); - try { - if (!State.NOT_STARTED.equals(state) && !State.FINISHED.equals(state)) { - spiderThread.stopScan(); - state = State.FINISHED; - SpiderEventPublisher.publishScanEvent( - ScanEventPublisher.SCAN_STOPPED_EVENT, this.scanId); - Stats.incCounter(SPIDER_SCAN_STOPPED_STATS); - } - } finally { - lock.unlock(); - } - } - - /** - * Returns the URLs found during the scan. - * - *

Note: Iterations must be {@code synchronized} on returned object. Failing - * to do so might result in {@code ConcurrentModificationException}. - * - * @return the URLs found during the scan - * @see ConcurrentModificationException - */ - public Set getResults() { - return foundURIs; - } - - /** - * Returns the resources found during the scan. - * - *

Note: Iterations must be {@code synchronized} on returned object. Failing - * to do so might result in {@code ConcurrentModificationException}. - * - * @return the resources found during the scan - * @see ConcurrentModificationException - */ - public List getResourcesFound() { - return resourcesFound; - } - - /** - * Returns the resources found during the scan that were not successfully obtained because of - * I/O errors. - * - *

Note: Iterations must be {@code synchronized} on returned object. Failing - * to do so might result in {@code ConcurrentModificationException}. - * - * @return the resources found during the scan that were not successfully obtained - * @since 2.6.0 - */ - public List getResourcesIoErrors() { - return resourcesIoErrors; - } - - /** - * Returns the URLs, out of scope, found during the scan. - * - *

Note: Iterations must be {@code synchronized} on returned object. Failing - * to do so might result in {@code ConcurrentModificationException}. - * - * @return the URLs, out of scope, found during the scan - * @see ConcurrentModificationException - */ - public Set getResultsOutOfScope() { - return foundURIsOutOfScope; - } - - @Override - public void notifySpiderTaskResult(org.zaproxy.zap.spider.SpiderTaskResult spiderTaskResult) { - HttpMessage msg = spiderTaskResult.getHttpMessage(); - HttpRequestHeader requestHeader = msg.getRequestHeader(); - HttpResponseHeader responseHeader = msg.getResponseHeader(); - SpiderResource resource = - new SpiderResource( - msg.getHistoryRef() != null ? msg.getHistoryRef().getHistoryId() : -1, - requestHeader.getMethod(), - requestHeader.getURI().toString(), - responseHeader.getStatusCode(), - responseHeader.getReasonPhrase(), - spiderTaskResult.isProcessed(), - spiderTaskResult.getReasonNotProcessed()); - - if (msg.isResponseFromTargetHost()) { - resourcesFound.add(resource); - Stats.incCounter(SPIDER_URL_FOUND_STATS); - } else { - resourcesIoErrors.add(resource); - Stats.incCounter(SPIDER_URL_ERROR_STATS); - } - - if (View.isInitialised()) { - addMessageToMessagesTableModel(spiderTaskResult); - } - } - - private void addMessageToMessagesTableModel( - final org.zaproxy.zap.spider.SpiderTaskResult spiderTaskResult) { - if (spiderTaskResult.getHttpMessage().getHistoryRef() == null) { - return; - } - - if (EventQueue.isDispatchThread() || cleared) { - if (cleared) { - return; - } - - if (messagesTableModel == null) { - messagesTableModel = new SpiderMessagesTableModel(); - } - messagesTableModel.addHistoryReference( - spiderTaskResult.getHttpMessage().getHistoryRef(), - spiderTaskResult.isProcessed(), - spiderTaskResult.getReasonNotProcessed()); - return; - } - - EventQueue.invokeLater( - new Runnable() { - - @Override - public void run() { - addMessageToMessagesTableModel(spiderTaskResult); - } - }); - } - - @Override - public void spiderComplete(boolean successful) { - lock.lock(); - try { - state = State.FINISHED; - SpiderEventPublisher.publishScanEvent( - ScanEventPublisher.SCAN_COMPLETED_EVENT, this.scanId); - } finally { - lock.unlock(); - } - if (listener != null) { - listener.scanFinshed(this.getScanId(), this.getDisplayName()); - } - } - - @Override - public void spiderProgress(int percentageComplete, int numberCrawled, int numberToCrawl) { - if (this.progress != percentageComplete) { - this.progress = percentageComplete; - SpiderEventPublisher.publishScanProgressEvent(scanId, percentageComplete); - } - - if (listener != null) { - listener.scanProgress(this.getScanId(), this.getDisplayName(), percentageComplete, 100); - } - } - - @Override - public void foundURI( - String uri, - String method, - org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus status) { - numberOfURIsFound.incrementAndGet(); - if (FETCH_STATUS_IN_SCOPE.contains(status)) { - foundURIs.add(uri); - } else if (FETCH_STATUS_OUT_OF_SCOPE.contains(status)) { - foundURIsOutOfScope.add(uri); - } - } - - @Override - public void run() { - // Nothing to do. - } - - @Override - public void setScanId(int id) { - this.scanId = id; - } - - @Override - public void setDisplayName(String name) { - this.displayName = name; - } - - @Override - public String getDisplayName() { - return this.displayName; - } - - @Override - public boolean isStopped() { - return this.spiderThread.isStopped(); - } - - @Override - public int getMaximum() { - return 100; - } - - /** - * Gets the number of URIs, in and out of scope, found during the scan. - * - * @return the number of URIs found during the scan - * @since 2.4.3 - */ - public int getNumberOfURIsFound() { - return numberOfURIsFound.get(); - } - - public int getNumberOfNodesAdded() { - return this.spiderThread.getNumberOfNodesAdded(); - } - - @Override - public boolean isPaused() { - return this.spiderThread.isPaused(); - } - - @Override - public boolean isRunning() { - return this.spiderThread.isRunning(); - } - - @Override - public void scanFinshed(String host) { - this.spiderComplete(true); - Stats.incCounter(SPIDER_SCAN_TIME_STATS, this.spiderThread.getTimeTakenInMs()); - } - - @Override - public void scanProgress(String host, int progress, int maximum) {} - - public TableModel getResultsTableModel() { - return this.spiderThread.getResultsTableModel(); - } - - public SpiderPanelTableModel getAddedNodesTableModel() { - return this.spiderThread.getAddedNodesTableModel(); - } - - /** - * Gets the {@code TableModel} of the messages sent during the spidering process. - * - * @return a {@code TableModel} with the messages sent - * @since 2.5.0 - */ - TableModel getMessagesTableModel() { - if (messagesTableModel == null) { - messagesTableModel = new SpiderMessagesTableModel(); - } - return messagesTableModel; - } - - public void setListener(ScanListenner2 listener) { - this.listener = listener; - } - - public void setCustomSpiderParsers( - List customSpiderParsers) { - spiderThread.setCustomSpiderParsers(customSpiderParsers); - } - - public void setCustomFetchFilters( - List customFetchFilters) { - spiderThread.setCustomFetchFilters(customFetchFilters); - } - - public void setCustomParseFilters( - List customParseFilters) { - spiderThread.setCustomParseFilters(customParseFilters); - } - - /** - * Clears the table model of the HTTP messages sent. - * - * @since 2.5.0 - * @see #getMessagesTableModel() - */ - void clear() { - cleared = true; - if (messagesTableModel != null) { - messagesTableModel.clear(); - messagesTableModel = null; - } - } -} diff --git a/zap/src/main/java/org/zaproxy/zap/extension/spider/SpiderScanController.java b/zap/src/main/java/org/zaproxy/zap/extension/spider/SpiderScanController.java deleted file mode 100644 index bba3d0ae0ec..00000000000 --- a/zap/src/main/java/org/zaproxy/zap/extension/spider/SpiderScanController.java +++ /dev/null @@ -1,352 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2014 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.extension.spider; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; -import org.apache.commons.httpclient.URI; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.zaproxy.zap.model.ScanController; -import org.zaproxy.zap.model.Target; -import org.zaproxy.zap.users.User; - -/** - * @deprecated (2.12.0) See the spider add-on in zap-extensions instead. - */ -@Deprecated -public class SpiderScanController implements ScanController { - - private static final Logger log = LogManager.getLogger(SpiderScanController.class); - - private ExtensionSpider extension; - - /** - * The {@code Lock} for exclusive access of instance variables related to multiple active scans. - * - * @see #spiderScanMap - * @see #scanIdCounter - */ - private final Lock spiderScansLock; - - /** - * The counter used to give an unique ID to active scans. - * - *

Note: All accesses (both write and read) should be done while holding the - * {@code Lock} {@code spiderScansLock}. - * - * @see #spiderScansLock - * @see #startScan(String, Target, User, Object[]) - */ - private int scanIdCounter; - - /** - * A map that contains all {@code SpiderScan}s created (and not yet removed). Used to control - * (i.e. pause/resume and stop) the multiple active scans and get its results. The instance - * variable is never {@code null}. The map key is the ID of the scan. - * - *

Note: All accesses (both write and read) should be done while holding the - * {@code Lock} {@code spiderScansLock}. - * - * @see #spiderScansLock - * @see #startScan(String, Target, User, Object[]) - * @see #scanIdCounter - */ - private Map spiderScanMap; - - /** - * An ordered list of all of the {@code SpiderScan}s created (and not yet removed). Used to get - * provide the 'last' scan for client using the 'old' API that didn't support concurrent scans. - */ - private List spiderScanList; - - public SpiderScanController(ExtensionSpider extension) { - this.spiderScansLock = new ReentrantLock(); - this.extension = extension; - this.spiderScanMap = new HashMap<>(); - this.spiderScanList = new ArrayList<>(); - } - - @Override - public int startScan(String name, Target target, User user, Object[] contextSpecificObjects) { - spiderScansLock.lock(); - try { - int id = this.scanIdCounter++; - - org.zaproxy.zap.spider.SpiderParam spiderParams = extension.getSpiderParam(); - List customSpiderParsers = - new ArrayList<>(); - List customFetchFilters = new ArrayList<>(); - List customParseFilters = new ArrayList<>(); - URI startUri = null; - - if (contextSpecificObjects != null) { - for (Object obj : contextSpecificObjects) { - if (obj instanceof org.zaproxy.zap.spider.SpiderParam) { - log.debug("Setting custom spider params"); - spiderParams = (org.zaproxy.zap.spider.SpiderParam) obj; - } else if (obj instanceof org.zaproxy.zap.spider.parser.SpiderParser) { - customSpiderParsers.add((org.zaproxy.zap.spider.parser.SpiderParser) obj); - } else if (obj instanceof org.zaproxy.zap.spider.filters.FetchFilter) { - customFetchFilters.add((org.zaproxy.zap.spider.filters.FetchFilter) obj); - } else if (obj instanceof org.zaproxy.zap.spider.filters.ParseFilter) { - customParseFilters.add((org.zaproxy.zap.spider.filters.ParseFilter) obj); - } else if (obj instanceof URI) { - startUri = (URI) obj; - } else { - log.error( - "Unexpected contextSpecificObject: {}", - obj.getClass().getCanonicalName()); - } - } - } - - if (spiderParams.getMaxChildren() > 0) { - // Add the filters to filter on maximum number of children - org.zaproxy.zap.spider.filters.MaxChildrenFetchFilter maxChildrenFetchFilter = - new org.zaproxy.zap.spider.filters.MaxChildrenFetchFilter(); - maxChildrenFetchFilter.setMaxChildren(spiderParams.getMaxChildren()); - maxChildrenFetchFilter.setModel(extension.getModel()); - - org.zaproxy.zap.spider.filters.MaxChildrenParseFilter maxChildrenParseFilter = - new org.zaproxy.zap.spider.filters.MaxChildrenParseFilter( - extension.getMessages()); - maxChildrenParseFilter.setMaxChildren(spiderParams.getMaxChildren()); - maxChildrenParseFilter.setModel(extension.getModel()); - - customFetchFilters.add(maxChildrenFetchFilter); - customParseFilters.add(maxChildrenParseFilter); - } - - SpiderScan scan = - new SpiderScan(extension, spiderParams, target, startUri, user, id, name); - scan.setCustomSpiderParsers(customSpiderParsers); - scan.setCustomFetchFilters(customFetchFilters); - scan.setCustomParseFilters(customParseFilters); - - this.spiderScanMap.put(id, scan); - this.spiderScanList.add(scan); - scan.start(); - - return id; - } finally { - spiderScansLock.unlock(); - } - } - - @Override - public SpiderScan getScan(int id) { - return this.spiderScanMap.get(id); - } - - @Override - public SpiderScan getLastScan() { - spiderScansLock.lock(); - try { - if (spiderScanList.isEmpty()) { - return null; - } - return spiderScanList.get(spiderScanList.size() - 1); - } finally { - spiderScansLock.unlock(); - } - } - - @Override - public List getAllScans() { - List list = new ArrayList<>(); - spiderScansLock.lock(); - try { - for (SpiderScan scan : spiderScanList) { - list.add(scan); - } - return list; - } finally { - spiderScansLock.unlock(); - } - } - - @Override - public List getActiveScans() { - List list = new ArrayList<>(); - spiderScansLock.lock(); - try { - for (SpiderScan scan : spiderScanList) { - if (!scan.isStopped()) { - list.add(scan); - } - } - return list; - } finally { - spiderScansLock.unlock(); - } - } - - @Override - public SpiderScan removeScan(int id) { - spiderScansLock.lock(); - - try { - SpiderScan ascan = this.spiderScanMap.get(id); - if (!spiderScanMap.containsKey(id)) { - // throw new IllegalArgumentException("Unknown id " + id); - return null; - } - ascan.stopScan(); - ascan.clear(); - spiderScanMap.remove(id); - spiderScanList.remove(ascan); - return ascan; - } finally { - spiderScansLock.unlock(); - } - } - - public int getTotalNumberScans() { - return spiderScanMap.size(); - } - - @Override - public void stopAllScans() { - spiderScansLock.lock(); - try { - for (SpiderScan scan : spiderScanMap.values()) { - scan.stopScan(); - } - } finally { - spiderScansLock.unlock(); - } - } - - @Override - public void pauseAllScans() { - spiderScansLock.lock(); - try { - for (SpiderScan scan : spiderScanMap.values()) { - scan.pauseScan(); - } - } finally { - spiderScansLock.unlock(); - } - } - - @Override - public void resumeAllScans() { - spiderScansLock.lock(); - try { - for (SpiderScan scan : spiderScanMap.values()) { - scan.resumeScan(); - } - } finally { - spiderScansLock.unlock(); - } - } - - @Override - public int removeAllScans() { - spiderScansLock.lock(); - try { - int count = 0; - for (Iterator it = spiderScanMap.values().iterator(); it.hasNext(); ) { - SpiderScan ascan = it.next(); - ascan.stopScan(); - ascan.clear(); - it.remove(); - spiderScanList.remove(ascan); - count++; - } - return count; - } finally { - spiderScansLock.unlock(); - } - } - - @Override - public int removeFinishedScans() { - spiderScansLock.lock(); - try { - int count = 0; - for (Iterator it = spiderScanMap.values().iterator(); it.hasNext(); ) { - SpiderScan scan = it.next(); - if (scan.isStopped()) { - scan.stopScan(); - scan.clear(); - it.remove(); - spiderScanList.remove(scan); - count++; - } - } - return count; - } finally { - spiderScansLock.unlock(); - } - } - - @Override - public void stopScan(int id) { - spiderScansLock.lock(); - try { - if (this.spiderScanMap.containsKey(id)) { - this.spiderScanMap.get(id).stopScan(); - } - } finally { - spiderScansLock.unlock(); - } - } - - @Override - public void pauseScan(int id) { - spiderScansLock.lock(); - try { - if (this.spiderScanMap.containsKey(id)) { - this.spiderScanMap.get(id).pauseScan(); - } - } finally { - spiderScansLock.unlock(); - } - } - - @Override - public void resumeScan(int id) { - spiderScansLock.lock(); - try { - if (this.spiderScanMap.containsKey(id)) { - this.spiderScanMap.get(id).resumeScan(); - } - } finally { - spiderScansLock.unlock(); - } - } - - public void reset() { - this.removeAllScans(); - spiderScansLock.lock(); - try { - this.scanIdCounter = 0; - } finally { - spiderScansLock.unlock(); - } - } -} diff --git a/zap/src/main/java/org/zaproxy/zap/extension/spider/SpiderThread.java b/zap/src/main/java/org/zaproxy/zap/extension/spider/SpiderThread.java deleted file mode 100644 index ebf8109195b..00000000000 --- a/zap/src/main/java/org/zaproxy/zap/extension/spider/SpiderThread.java +++ /dev/null @@ -1,672 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2010 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.extension.spider; - -import java.awt.EventQueue; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Date; -import java.util.Enumeration; -import java.util.LinkedList; -import java.util.List; -import javax.swing.DefaultListModel; -import javax.swing.tree.TreeNode; -import org.apache.commons.httpclient.URI; -import org.apache.commons.httpclient.URIException; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.parosproxy.paros.Constant; -import org.parosproxy.paros.model.HistoryReference; -import org.parosproxy.paros.model.Model; -import org.parosproxy.paros.model.SiteNode; -import org.parosproxy.paros.network.HttpMessage; -import org.parosproxy.paros.view.View; -import org.zaproxy.zap.model.Context; -import org.zaproxy.zap.model.ScanListenner; -import org.zaproxy.zap.model.ScanThread; -import org.zaproxy.zap.model.SessionStructure; -import org.zaproxy.zap.model.StructuralNode; -import org.zaproxy.zap.model.TechSet; -import org.zaproxy.zap.users.User; - -/** - * The Class SpiderThread that controls the spidering process on a particular site. Being a - * ScanThread, it also handles the update of the graphical UI and any other "extension-level" - * required actions. - * - * @deprecated (2.12.0) See the spider add-on in zap-extensions instead. - */ -@Deprecated -public class SpiderThread extends ScanThread implements org.zaproxy.zap.spider.SpiderListener { - - /** Whether the scanning process has stopped (either completed, either by user request). */ - private boolean stopScan = false; - - /** Whether the scanning process is paused. */ - private boolean isPaused = false; - - /** Whether the scanning process is running. */ - private boolean isAlive = false; - - /** The related extension. */ - private ExtensionSpider extension; - - /** The spider. */ - private org.zaproxy.zap.spider.Spider spider = null; - - /** The pending spider listeners which will be added to the Spider as soon at is initialized. */ - private List pendingSpiderListeners; - - /** The spider done. */ - private int spiderDone = 0; - - /** The spider todo. It will be updated by the "spiderProgress()" method. */ - private int spiderTodo = 1; - - /** The Constant log used for logging. */ - private static final Logger log = LogManager.getLogger(SpiderThread.class); - - /** The just scan in scope. */ - private boolean justScanInScope = false; - - /** The scan children. */ - private boolean scanChildren = false; - - /** The scan context. */ - private Context scanContext = null; - - /** The scan user. */ - private User scanUser = null; - - /** The results model. */ - private SpiderPanelTableModel resultsModel; - - /** The added nodes model, i.e. the names of the nodes that were added to the sites tree. */ - private SpiderPanelTableModel addedNodesModel; - - /** The start uri. */ - private URI startURI = null; - - private org.zaproxy.zap.spider.SpiderParam spiderParams; - - private List customSpiderParsers = null; - - private List customFetchFilters = null; - - private List customParseFilters = null; - - private final String id; - - private Date started; - - private long timeTakenInMs; - - /** - * Constructs a {@code SpiderThread} with the given data. - * - * @param id the ID of the spider, usually a unique integer - * @param extension the extension to obtain configurations and notify the view - * @param spiderParams the spider options - * @param site the name that identifies the target site - * @param listenner the scan listener - * @since 2.6.0 - */ - public SpiderThread( - String id, - ExtensionSpider extension, - org.zaproxy.zap.spider.SpiderParam spiderParams, - String site, - ScanListenner listenner) { - super(site, listenner); - log.debug("Initializing spider thread for site: {}", site); - this.id = id; - this.extension = extension; - this.site = site; - this.pendingSpiderListeners = new LinkedList<>(); - this.resultsModel = extension.getView() != null ? new SpiderPanelTableModel() : null; - // This can be used in daemon mode via the API - this.addedNodesModel = new SpiderPanelTableModel(false); - this.spiderParams = spiderParams; - - setName("ZAP-SpiderInitThread-" + id); - } - - @Override - public void run() { - try { - runScan(); - } catch (Exception e) { - log.error("An error occurred while starting the spider:", e); - stopScan(); - } - } - - /** Runs the scan. */ - private void runScan() { - // Do the scan - spiderDone = 0; - started = new Date(); - log.info("Starting spidering scan on {} at {}", site, started); - startSpider(); - this.isAlive = true; - } - - @Override - public void stopScan() { - if (spider != null) { - spider.stop(); - } - stopScan = true; - isAlive = false; - this.listenner.scanFinshed(site); - } - - @Override - public boolean isStopped() { - return stopScan; - } - - @Override - public boolean isRunning() { - return isAlive; - } - - @Override - public DefaultListModel getList() { - // Not used, as the SpiderPanel is relying on a TableModel - return null; - } - - @Override - public void pauseScan() { - if (spider != null) { - spider.pause(); - } - this.isPaused = true; - } - - @Override - public void resumeScan() { - if (spider != null) { - spider.resume(); - } - this.isPaused = false; - } - - @Override - public boolean isPaused() { - return this.isPaused; - } - - @Override - public int getMaximum() { - return this.spiderDone + this.spiderTodo; - } - - /** Start spider. */ - private void startSpider() { - - spider = - new org.zaproxy.zap.spider.Spider( - id, extension, spiderParams, extension.getModel(), this.scanContext); - - // Register this thread as a Spider Listener, so it gets notified of events and is able - // to manipulate the UI accordingly - spider.addSpiderListener(this); - - // Add the pending listeners - for (org.zaproxy.zap.spider.SpiderListener l : pendingSpiderListeners) { - spider.addSpiderListener(l); - } - - // Add the list of (regex) URIs that should be excluded - List excludeList = new ArrayList<>(); - excludeList.addAll(extension.getExcludeList()); - excludeList.addAll(extension.getModel().getSession().getExcludeFromSpiderRegexs()); - excludeList.addAll(extension.getModel().getSession().getGlobalExcludeURLRegexs()); - spider.setExcludeList(excludeList); - - // Add seeds accordingly - addSeeds(); - - spider.setScanAsUser(scanUser); - - // Add any custom parsers and filters specified - if (this.customSpiderParsers != null) { - for (org.zaproxy.zap.spider.parser.SpiderParser sp : this.customSpiderParsers) { - spider.addCustomParser(sp); - } - } - if (this.customFetchFilters != null) { - for (org.zaproxy.zap.spider.filters.FetchFilter ff : this.customFetchFilters) { - spider.addFetchFilter(ff); - } - } - if (this.customParseFilters != null) { - for (org.zaproxy.zap.spider.filters.ParseFilter pf : this.customParseFilters) { - spider.addParseFilter(pf); - } - } - - // Start the spider - spider.start(); - } - - /** - * Adds the initial seeds, with following constraints: - * - *

    - *
  • If a {@link #scanContext context} is provided: - *
      - *
    • {@link #startURI Start URI}, if in context; - *
    • {@link #startNode Start node}, if in context; - *
    • All nodes in the context; - *
    - *
  • If spidering just in {@link #justScanInScope scope}: - *
      - *
    • Start URI, if in scope; - *
    • Start node, if in scope; - *
    • All nodes in scope; - *
    - *
  • If there's no context/scope restriction: - *
      - *
    • Start URI; - *
    • Start node, also: - *
        - *
      • Child nodes, if {@link #scanChildren spidering "recursively"}. - *
      - *
    - *
- * - * @see #addStartSeeds() - */ - private void addSeeds() { - addStartSeeds(); - - List nodesInScope = Collections.emptyList(); - if (this.scanContext != null) { - log.debug("Adding seed for Scan of all in context {}", scanContext.getName()); - nodesInScope = this.scanContext.getNodesInContextFromSiteTree(); - } else if (justScanInScope) { - log.debug("Adding seed for Scan of all in scope."); - nodesInScope = Model.getSingleton().getSession().getNodesInScopeFromSiteTree(); - } - - if (!nodesInScope.isEmpty()) { - for (SiteNode node : nodesInScope) { - addSeed(node); - } - } - } - - /** - * Adds the start seeds ({@link #startNode start node} and {@link #startURI start URI}) to the - * spider. - * - * @see #addSeeds() - */ - private void addStartSeeds() { - if (scanContext != null) { - if (startNode != null && scanContext.isInContext(startNode)) { - addSeed(startNode); - } - if (startURI != null && scanContext.isInContext(startURI.toString())) { - spider.addSeed(startURI); - } - return; - } - - if (justScanInScope) { - if (startNode != null && Model.getSingleton().getSession().isInScope(startNode)) { - addSeed(startNode); - } - if (startURI != null - && Model.getSingleton().getSession().isInScope(startURI.toString())) { - spider.addSeed(startURI); - } - return; - } - - if (startNode != null) { - addSeeds(startNode); - } - if (startURI != null) { - spider.addSeed(startURI); - } - } - - /** - * Adds the given node as seed, if the corresponding message is not an image. - * - * @param node the node that will be added as seed - */ - private void addSeed(SiteNode node) { - try { - if (!node.isRoot() && node.getHistoryReference() != null) { - HttpMessage msg = node.getHistoryReference().getHttpMessage(); - if (!msg.getResponseHeader().isImage()) { - spider.addSeed(msg); - } - } - } catch (Exception e) { - log.error("Error while adding seed for Spider scan: {}", e.getMessage(), e); - } - } - - /** - * Adds as seeds the given node and, if {@link #scanChildren} is {@code true}, the children - * nodes. - * - * @param node the node that will be added as seed and possible the children nodes - */ - private void addSeeds(SiteNode node) { - // Add the current node - addSeed(node); - - // If the "scanChildren" option is enabled, add them - if (scanChildren) { - @SuppressWarnings("unchecked") - Enumeration en = node.children(); - while (en.hasMoreElements()) { - SiteNode sn = (SiteNode) en.nextElement(); - addSeeds(sn); - } - } - } - - @Override - public void spiderComplete(boolean successful) { - Date finished = new Date(); - log.info("Spider scanning complete: {} on {} at {}", successful, site, finished); - this.timeTakenInMs = finished.getTime() - started.getTime(); - stopScan = true; - this.isAlive = false; - this.listenner.scanFinshed(site); - } - - /** - * Returns the time taken in milliseconds. This will be total time taken if the scan has - * finished or the time taken so far if it is still running. - * - * @return the time taken in milliseconds - * @since 2.11.0 - */ - public long getTimeTakenInMs() { - if (this.timeTakenInMs > 0) { - return this.timeTakenInMs; - } - if (this.started != null) { - return System.currentTimeMillis() - started.getTime(); - } - return 0; - } - - @Override - public void foundURI( - String uri, - String method, - org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus status) { - if (resultsModel != null) { - addUriToResultsModel(uri, method, status); - } - } - - private void addUriToResultsModel( - final String uri, - final String method, - final org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus status) { - if (!EventQueue.isDispatchThread()) { - EventQueue.invokeLater( - new Runnable() { - - @Override - public void run() { - addUriToResultsModel(uri, method, status); - } - }); - return; - } - - // Add the new result - if (status == org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus.VALID) { - resultsModel.addScanResult(uri, method, null, false); - } else { - resultsModel.addScanResult( - uri, - method, - getStatusLabel(status), - status != org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus.SEED); - } - - // Update the count of found URIs - extension.getSpiderPanel().updateFoundCount(); - } - - private void addUriToAddedNodesModel( - final String uri, final String method, final String params) { - if (!EventQueue.isDispatchThread()) { - EventQueue.invokeLater( - new Runnable() { - - @Override - public void run() { - addUriToAddedNodesModel(uri, method, params); - } - }); - return; - } - - // Add the new result - addedNodesModel.addScanResult(uri, method, null, false); - - if (extension.getView() != null) { - // Update the count of added nodes - extension.getSpiderPanel().updateAddedCount(); - } - } - - private String getStatusLabel(org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus status) { - switch (status) { - case SEED: - return Constant.messages.getString("spider.table.flags.seed"); - case OUT_OF_CONTEXT: - return Constant.messages.getString("spider.table.flags.outofcontext"); - case OUT_OF_SCOPE: - return Constant.messages.getString("spider.table.flags.outofscope"); - case ILLEGAL_PROTOCOL: - return Constant.messages.getString("spider.table.flags.illegalprotocol"); - case USER_RULES: - return Constant.messages.getString("spider.table.flags.userrules"); - default: - return status.toString(); - } - } - - @Override - public void notifySpiderTaskResult(org.zaproxy.zap.spider.SpiderTaskResult spiderTaskResult) { - // Add the read message to the Site Map (tree or db structure) - try { - HttpMessage msg = spiderTaskResult.getHttpMessage(); - int type = - msg.isResponseFromTargetHost() - ? HistoryReference.TYPE_SPIDER - : HistoryReference.TYPE_SPIDER_TEMPORARY; - HistoryReference historyRef = - new HistoryReference(extension.getModel().getSession(), type, msg); - - if (msg.isResponseFromTargetHost()) { - addMessageToSitesTree(historyRef, msg); - } - } catch (Exception e) { - log.error(e.getMessage(), e); - } - } - - /** - * Adds the given message to the sites tree. - * - * @param historyReference the history reference of the message, must not be {@code null} - * @param message the actual message, must not be {@code null} - */ - private void addMessageToSitesTree( - final HistoryReference historyReference, final HttpMessage message) { - if (View.isInitialised() && !EventQueue.isDispatchThread()) { - EventQueue.invokeLater( - new Runnable() { - - @Override - public void run() { - addMessageToSitesTree(historyReference, message); - } - }); - return; - } - - StructuralNode node = - SessionStructure.addPath(Model.getSingleton(), historyReference, message, true); - if (node != null) { - try { - addUriToAddedNodesModel( - SessionStructure.getNodeName(Model.getSingleton(), message), - message.getRequestHeader().getMethod(), - ""); - } catch (URIException e) { - log.error("Error while adding node to added nodes model: {}", e.getMessage(), e); - } - } - } - - @Override - public void spiderProgress( - final int percentageComplete, final int numberCrawled, final int numberToCrawl) { - this.spiderDone = numberCrawled; - this.spiderTodo = numberToCrawl; - this.scanProgress(site, numberCrawled, numberCrawled + numberToCrawl); - } - - @Override - public SiteNode getStartNode() { - return startNode; - } - - @Override - public void setStartNode(SiteNode startNode) { - this.startNode = startNode; - } - - /** - * Sets the start uri. This will be used if no startNode is identified. - * - * @param startURI the new start uri - */ - public void setStartURI(URI startURI) { - this.startURI = startURI; - } - - @Override - public void reset() { - if (resultsModel != null) { - this.resultsModel.removeAllElements(); - } - if (addedNodesModel != null) { - this.addedNodesModel.removeAllElements(); - } - } - - /** - * Adds a new spider listener. - * - * @param listener the listener - */ - public void addSpiderListener(org.zaproxy.zap.spider.SpiderListener listener) { - if (spider != null) { - this.spider.addSpiderListener(listener); - } else { - this.pendingSpiderListeners.add(listener); - } - } - - @Override - public void setJustScanInScope(boolean scanInScope) { - this.justScanInScope = scanInScope; - } - - @Override - public boolean getJustScanInScope() { - return justScanInScope; - } - - @Override - public void setScanChildren(boolean scanChildren) { - this.scanChildren = scanChildren; - } - - @Override - public int getProgress() { - return this.progress; - } - - /** - * Gets the results table model. - * - * @return the results table model - */ - public SpiderPanelTableModel getResultsTableModel() { - return this.resultsModel; - } - - public SpiderPanelTableModel getAddedNodesTableModel() { - return this.addedNodesModel; - } - - @Override - public void setScanContext(Context context) { - this.scanContext = context; - } - - @Override - public void setScanAsUser(User user) { - this.scanUser = user; - } - - @Override - public void setTechSet(TechSet techSet) { - // Ignore - } - - public void setCustomSpiderParsers( - List customSpiderParsers) { - this.customSpiderParsers = customSpiderParsers; - } - - public void setCustomFetchFilters( - List customFetchFilters) { - this.customFetchFilters = customFetchFilters; - } - - public void setCustomParseFilters( - List customParseFilters) { - this.customParseFilters = customParseFilters; - } - - public int getNumberOfNodesAdded() { - return getAddedNodesTableModel().getRowCount(); - } -} diff --git a/zap/src/main/java/org/zaproxy/zap/spider/DomainAlwaysInScopeMatcher.java b/zap/src/main/java/org/zaproxy/zap/spider/DomainAlwaysInScopeMatcher.java deleted file mode 100644 index 75e80b5da24..00000000000 --- a/zap/src/main/java/org/zaproxy/zap/spider/DomainAlwaysInScopeMatcher.java +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2014 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.spider; - -import java.util.regex.Pattern; -import org.zaproxy.zap.utils.Enableable; - -/** - * Class that contains rules to checks if a domain is in scope. - * - *

It supports both plain text and regular expression checks. - * - * @see #matches(String) - * @deprecated (2.12.0) See the spider add-on in zap-extensions instead. - */ -@Deprecated -public class DomainAlwaysInScopeMatcher extends Enableable { - - private final Pattern pattern; - private final String domain; - private final boolean regex; - - public DomainAlwaysInScopeMatcher(Pattern pattern) { - super(true); - - if (pattern == null) { - throw new IllegalArgumentException("Parameter pattern must not be null."); - } - - this.pattern = pattern; - this.regex = true; - this.domain = null; - } - - public DomainAlwaysInScopeMatcher(String domain) { - super(true); - - if (domain == null || domain.isEmpty()) { - throw new IllegalArgumentException("Parameter domain must not be null or empty."); - } - - this.domain = domain; - this.regex = false; - this.pattern = null; - } - - public DomainAlwaysInScopeMatcher(DomainAlwaysInScopeMatcher other) { - super(other.isEnabled()); - - this.domain = other.domain; - this.regex = other.regex; - this.pattern = other.pattern; - } - - public String getValue() { - if (isRegex()) { - return pattern.pattern(); - } - - return domain; - } - - public boolean isRegex() { - return regex; - } - - /** - * Tells whether or not the given domain is considered always in scope. - * - * @param domain the domain that will be checked - * @return {@code true} if the domain is considered always in scope, {@code false} otherwise. - */ - public boolean matches(String domain) { - if (pattern != null) { - return pattern.matcher(domain).matches(); - } - - return this.domain.equals(domain); - } - - @Override - public int hashCode() { - final int prime = 31; - int result = super.hashCode(); - result = prime * result + ((domain == null) ? 0 : domain.hashCode()); - result = prime * result + ((pattern == null) ? 0 : pattern.pattern().hashCode()); - return result; - } - - @Override - public boolean equals(Object object) { - if (this == object) { - return true; - } - if (!super.equals(object)) { - return false; - } - if (getClass() != object.getClass()) { - return false; - } - DomainAlwaysInScopeMatcher other = (DomainAlwaysInScopeMatcher) object; - if (domain == null) { - if (other.domain != null) { - return false; - } - } else if (!domain.equals(other.domain)) { - return false; - } - if (pattern == null) { - if (other.pattern != null) { - return false; - } - } else if (!pattern.pattern().equals(other.pattern.pattern())) { - return false; - } - return true; - } - - public static Pattern createPattern(String regex) { - return Pattern.compile(regex, Pattern.CASE_INSENSITIVE); - } -} diff --git a/zap/src/main/java/org/zaproxy/zap/spider/Spider.java b/zap/src/main/java/org/zaproxy/zap/spider/Spider.java deleted file mode 100644 index f83854bf5e9..00000000000 --- a/zap/src/main/java/org/zaproxy/zap/spider/Spider.java +++ /dev/null @@ -1,869 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2012 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.spider; - -import java.util.Iterator; -import java.util.LinkedHashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.locks.Condition; -import java.util.concurrent.locks.ReentrantLock; -import java.util.regex.Pattern; -import org.apache.commons.httpclient.URI; -import org.apache.commons.httpclient.URIException; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.parosproxy.paros.model.Model; -import org.parosproxy.paros.network.HttpMessage; -import org.parosproxy.paros.network.HttpRequestHeader; -import org.parosproxy.paros.network.HttpSender; -import org.zaproxy.zap.model.Context; -import org.zaproxy.zap.users.User; - -/** - * The Class Spider. - * - * @deprecated (2.12.0) See the spider add-on in zap-extensions instead. - */ -@Deprecated -public class Spider { - - /** The spider parameters. */ - private SpiderParam spiderParam; - - /** The model. */ - private Model model; - - /** The listeners for Spider related events. */ - private List listeners; - - /** If the spider is currently paused. */ - private volatile boolean paused; - - /** The spider is currently stopped. */ - private volatile boolean stopped; - - /** The pause lock, used for locking access to the "paused" variable. */ - private ReentrantLock pauseLock = new ReentrantLock(); - - /** The controller that manages the spidering process. */ - private SpiderController controller; - - /** - * The condition that is used for the threads in the pool to wait on, when the Spider crawling - * is paused. When the Spider is resumed, all the waiting threads are awakened. - */ - private Condition pausedCondition = pauseLock.newCondition(); - - /** The thread pool for spider workers. */ - private ExecutorService threadPool; - - /** The default fetch filter. */ - private org.zaproxy.zap.spider.filters.DefaultFetchFilter defaultFetchFilter; - - /** The seed list. */ - private LinkedHashSet seedList; - - /** The extension. */ - private org.zaproxy.zap.extension.spider.ExtensionSpider extension; - - /** The Constant log. */ - private static final Logger log = LogManager.getLogger(Spider.class); - - /** The HTTP sender used to effectively send the data. */ - private HttpSender httpSender; - - /** The count of the tasks finished. */ - private int tasksDoneCount; - - /** The total count of all the submitted tasks. */ - private int tasksTotalCount; - - /** The scan context. If null, the scan is not performed in a context. */ - private Context scanContext; - - /** The scan user. */ - private User scanUser; - - /** The time the scan was started */ - private long timeStarted; - - /** - * The initialized marks if the spidering process is completely started. It solves the problem - * when the first task is processed and the process is finished before the other seeds are - * added. - */ - private boolean initialized; - - /** - * we do not want to recurse into an SVN folder, or a subfolder of an SVN folder, if one was - * created from a previous Spider run - */ - private static final Pattern svnUrlPattern = Pattern.compile("\\.svn/"); // case sensitive - - /** - * we do not want to recurse into a Git folder, or a subfolder of a Git folder, if one was - * created from a previous Spider run - */ - private static final Pattern gitUrlPattern = Pattern.compile("\\.git/"); // case sensitive - - private final String id; - - /** - * Constructs a {@code Spider} with the given data. - * - * @param id the ID of the spider, usually a unique integer - * @param extension the extension - * @param spiderParam the spider param - * @param connectionParam the connection param - * @param model the model - * @param scanContext if a scan context is set, only URIs within the context are fetched and - * processed - * @since 2.6.0 - * @deprecated (2.12.0) Use {@code #Spider(String, ExtensionSpider, SpiderParam, Model, - * Context)} instead. - */ - @Deprecated - public Spider( - String id, - org.zaproxy.zap.extension.spider.ExtensionSpider extension, - SpiderParam spiderParam, - org.parosproxy.paros.network.ConnectionParam connectionParam, - Model model, - Context scanContext) { - this(id, extension, spiderParam, model, scanContext); - } - - /** - * Constructs a {@code Spider} with the given data. - * - * @param id the ID of the spider, usually a unique integer - * @param extension the extension - * @param spiderParam the spider param - * @param model the model - * @param scanContext if a scan context is set, only URIs within the context are fetched and - * processed - * @since 2.12.0 - */ - public Spider( - String id, - org.zaproxy.zap.extension.spider.ExtensionSpider extension, - SpiderParam spiderParam, - Model model, - Context scanContext) { - super(); - log.info("Spider initializing..."); - this.id = id; - this.spiderParam = spiderParam; - this.model = model; - this.extension = extension; - this.controller = new SpiderController(this, extension.getCustomParsers()); - this.listeners = new LinkedList<>(); - this.seedList = new LinkedHashSet<>(); - this.scanContext = scanContext; - - init(); - } - - /** Initialize the spider. */ - private void init() { - this.paused = false; - this.stopped = true; - this.tasksDoneCount = 0; - this.tasksTotalCount = 0; - this.initialized = false; - - // Add a default fetch filter and any custom ones - defaultFetchFilter = new org.zaproxy.zap.spider.filters.DefaultFetchFilter(); - this.addFetchFilter(defaultFetchFilter); - - for (org.zaproxy.zap.spider.filters.FetchFilter filter : - extension.getCustomFetchFilters()) { - this.addFetchFilter(filter); - } - - // Add a default parse filter and any custom ones - controller.setDefaultParseFilter( - new org.zaproxy.zap.spider.filters.DefaultParseFilter( - spiderParam, extension.getMessages())); - for (org.zaproxy.zap.spider.filters.ParseFilter filter : extension.getCustomParseFilters()) - this.addParseFilter(filter); - - // Add the scan context, if any - defaultFetchFilter.setScanContext(this.scanContext); - defaultFetchFilter.setDomainsAlwaysInScope(spiderParam.getDomainsAlwaysInScopeEnabled()); - } - - /* SPIDER Related */ - /** - * Adds a new seed for the Spider. - * - * @param msg the message used for seed. The request URI is used from the Request Header - */ - public void addSeed(HttpMessage msg) { - URI uri = msg.getRequestHeader().getURI(); - addSeed(uri); - } - - /** - * Adds a new seed for the Spider. - * - * @param uri the uri - */ - public void addSeed(URI uri) { - // Update the scope of the spidering process - String host = null; - - try { - host = uri.getHost(); - defaultFetchFilter.addScopeRegex(host); - } catch (URIException e) { - log.error("There was an error while adding seed value: {}", uri, e); - return; - } - // Add the seed to the list -- it will be added to the task list only when the spider is - // started - this.seedList.add(uri); - // Add the appropriate 'robots.txt' as a seed - if (getSpiderParam().isParseRobotsTxt()) { - addRootFileSeed(uri, "robots.txt"); - } - // Add the appropriate 'sitemap.xml' as a seed - if (getSpiderParam().isParseSitemapXml()) { - addRootFileSeed(uri, "sitemap.xml"); - } - // And add '.svn/entries' as a seed, for SVN based spidering - if (getSpiderParam().isParseSVNEntries()) { - addFileSeed(uri, ".svn/entries", svnUrlPattern); - addFileSeed(uri, ".svn/wc.db", svnUrlPattern); - } - - // And add '.git/index' as a seed, for Git based spidering - if (getSpiderParam().isParseGit()) { - addFileSeed(uri, ".git/index", gitUrlPattern); - } - } - - /** - * Adds a file seed, with the given file name, at the root of the base URI. - * - *

For example, with base URI as {@code http://example.com/some/path/file.html} and file name - * as {@code sitemap.xml} it's added the seed {@code http://example.com/sitemap.xml}. - * - * @param baseUri the base URI. - * @param fileName the file name. - */ - private void addRootFileSeed(URI baseUri, String fileName) { - String seed = - buildUri( - baseUri.getScheme(), - baseUri.getRawHost(), - baseUri.getPort(), - "/" + fileName); - try { - this.seedList.add(new URI(seed, true)); - } catch (Exception e) { - log.warn("Error while creating [{}] seed: {}", fileName, seed, e); - } - } - - /** - * Creates a URI (string) with the given scheme, host, port and path. The port is only added if - * not the default for the given scheme. - * - * @param scheme the scheme, {@code http} or {@code https}. - * @param host the name of the host. - * @param port the port. - * @param path the path, should start with {@code /}. - * @return the URI with the provided components. - */ - private static String buildUri(String scheme, char[] host, int port, String path) { - StringBuilder strBuilder = new StringBuilder(150); - strBuilder.append(scheme).append("://").append(host); - if (!isDefaultPort(scheme, port)) { - strBuilder.append(':').append(port); - } - strBuilder.append(path); - return strBuilder.toString(); - } - - /** - * Adds a file seed using the given base URI, file name and condition. - * - *

The file is added as part of the path, without existing file name. For example, with base - * URI as {@code http://example.com/some/path/file.html} and file name as {@code .git/index} - * it's added the seed {@code http://example.com/some/path/.git/index}. - * - *

If the given condition matches the base URI's path without the file name, the file seed is - * not added (this prevents adding the seed once again). - * - * @param baseUri the base URI to construct the file seed. - * @param fileName the name of the file seed. - * @param condition the condition to add the file seed. - */ - private void addFileSeed(URI baseUri, String fileName, Pattern condition) { - String fullpath = baseUri.getEscapedPath(); - if (fullpath == null) { - fullpath = ""; - } - - String name = baseUri.getEscapedName(); - if (name == null) { - name = ""; - } - - String pathminusfilename = fullpath.substring(0, fullpath.lastIndexOf(name)); - if (pathminusfilename.isEmpty()) { - pathminusfilename = "/"; - } - - if (condition.matcher(pathminusfilename).find()) { - return; - } - - String uri = - buildUri( - baseUri.getScheme(), - baseUri.getRawHost(), - baseUri.getPort(), - pathminusfilename + fileName); - try { - this.seedList.add(new URI(uri, true)); - } catch (Exception e) { - log.warn( - "Error while creating a seed URI for file [{}] from [{}] using [{}]:", - fileName, - baseUri, - uri, - e); - } - } - - /** - * Tells whether or not the given port is the default for the given scheme. - * - *

Only intended to be used with HTTP/S schemes. - * - * @param scheme the scheme. - * @param port the port. - * @return {@code true} if the given port is the default for the given scheme, {@code false} - * otherwise. - */ - private static boolean isDefaultPort(String scheme, int port) { - if (port == -1) { - return true; - } - - if ("http".equalsIgnoreCase(scheme)) { - return port == 80; - } - - if ("https".equalsIgnoreCase(scheme)) { - return port == 443; - } - - return false; - } - - /** - * Sets the exclude list which contains a List of strings, defining the uris that should be - * excluded. - * - * @param excludeList the new exclude list - */ - public void setExcludeList(List excludeList) { - log.debug("New Exclude list: {}", excludeList); - defaultFetchFilter.setExcludeRegexes(excludeList); - } - - /** - * Adds a new fetch filter to the spider. - * - * @param filter the filter - */ - public void addFetchFilter(org.zaproxy.zap.spider.filters.FetchFilter filter) { - controller.addFetchFilter(filter); - } - - /** - * Adds a new parse filter to the spider. - * - * @param filter the filter - */ - public void addParseFilter(org.zaproxy.zap.spider.filters.ParseFilter filter) { - controller.addParseFilter(filter); - } - - /** - * Gets the http sender. Can be called from the SpiderTask. - * - * @return the http sender - */ - protected HttpSender getHttpSender() { - return httpSender; - } - - /** - * Gets the spider parameters. Can be called from the SpiderTask. - * - * @return the spider parameters - */ - protected SpiderParam getSpiderParam() { - return spiderParam; - } - - /** - * Gets the controller. - * - * @return the controller - */ - protected SpiderController getController() { - return controller; - } - - /** - * Gets the model. - * - * @return the model - */ - protected Model getModel() { - return this.model; - } - - /** - * Submit a new task to the spidering task pool. - * - * @param task the task - */ - protected synchronized void submitTask(SpiderTask task) { - if (isStopped()) { - log.debug("Submitting task skipped ({}) as the Spider process is stopped.", task); - return; - } - if (isTerminated()) { - log.debug("Submitting task skipped ({}) as the Spider process is terminated.", task); - return; - } - this.tasksTotalCount++; - try { - this.threadPool.execute(task); - } catch (RejectedExecutionException e) { - log.debug( - "Submitted task was rejected ({}), spider state: [stopped={}, terminated={}].", - task, - isStopped(), - isTerminated()); - } - } - - /** - * Gets the extension. - * - * @return the extension - */ - protected org.zaproxy.zap.extension.spider.ExtensionSpider getExtensionSpider() { - return this.extension; - } - - /* SPIDER PROCESS maintenance - pause, resume, shutdown, etc. */ - - /** Starts the Spider crawling. */ - public void start() { - - log.info("Starting spider..."); - - this.timeStarted = System.currentTimeMillis(); - - fetchFilterSeeds(); - - // Check if seeds are available, otherwise the Spider will start, but will not have any - // seeds and will not stop. - if (seedList == null || seedList.isEmpty()) { - log.warn("No seeds available for the Spider. Cancelling scan..."); - notifyListenersSpiderComplete(false); - notifyListenersSpiderProgress(100, 0, 0); - return; - } - - if (scanUser != null) - log.info( - "Scan will be performed from the point of view of User: {}", - scanUser.getName()); - - this.controller.init(); - this.stopped = false; - this.paused = false; - this.initialized = false; - - // Initialize the thread pool - this.threadPool = - Executors.newFixedThreadPool( - spiderParam.getThreadCount(), - new SpiderThreadFactory("ZAP-SpiderThreadPool-" + id + "-thread-")); - - // Initialize the HTTP sender - httpSender = new HttpSender(HttpSender.SPIDER_INITIATOR); - httpSender.setUseGlobalState( - httpSender.isGlobalStateEnabled() || !spiderParam.isAcceptCookies()); - - // Do not follow redirections because the request is not updated, the redirections will be - // handled manually. - httpSender.setFollowRedirect(false); - - // Add the seeds - for (URI uri : seedList) { - log.debug("Adding seed for spider: {}", uri); - controller.addSeed(uri, HttpRequestHeader.GET); - } - // Mark the process as completely initialized - initialized = true; - } - - /** - * Filters the seed list using the current fetch filters, preventing any non-valid seed from - * being accessed. - * - * @see #seedList - * @see org.zaproxy.zap.spider.filters.FetchFilter - * @see SpiderController#getFetchFilters() - * @since 2.5.0 - */ - private void fetchFilterSeeds() { - if (seedList == null || seedList.isEmpty()) { - return; - } - - for (Iterator it = seedList.iterator(); it.hasNext(); ) { - URI seed = it.next(); - for (org.zaproxy.zap.spider.filters.FetchFilter filter : controller.getFetchFilters()) { - org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus filterReason = - filter.checkFilter(seed); - if (filterReason != org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus.VALID) { - log.debug("Seed: {} was filtered with reason: {}", seed, filterReason); - it.remove(); - break; - } - } - } - } - - /** Stops the Spider crawling. Must not be called from any of the threads in the thread pool. */ - public void stop() { - if (stopped) { - return; - } - this.stopped = true; - log.info("Stopping spidering process by request."); - - if (this.paused) { - // Have to resume first or we get a deadlock - this.resume(); - } - - // Issue the shutdown command - this.threadPool.shutdown(); - try { - if (!this.threadPool.awaitTermination(2, TimeUnit.SECONDS)) { - log.warn( - "Failed to await for all spider threads to stop in the given time (2s)..."); - for (Runnable task : this.threadPool.shutdownNow()) { - ((SpiderTask) task).cleanup(); - } - } - } catch (InterruptedException ignore) { - log.warn("Interrupted while awaiting for all spider threads to stop..."); - } - httpSender = null; - - // Notify the controller to clean up memory - controller.reset(); - this.threadPool = null; - - // Notify the listeners -- in the meanwhile - notifyListenersSpiderComplete(false); - } - - /** The Spidering process is complete. */ - private void complete() { - if (stopped) { - return; - } - - log.info("Spidering process is complete. Shutting down..."); - this.stopped = true; - httpSender = null; - - // Notify the controller to clean up memory - controller.reset(); - - // Issue the shutdown command on a separate thread, as the current thread is most likely one - // from the pool - new Thread( - new Runnable() { - @Override - public void run() { - if (threadPool != null) { - threadPool.shutdown(); - } - // Notify the listeners -- in the meanwhile - notifyListenersSpiderComplete(true); - controller.reset(); - threadPool = null; - } - }, - "ZAP-SpiderShutdownThread-" + id) - .start(); - } - - /** Pauses the Spider crawling. */ - public void pause() { - pauseLock.lock(); - try { - paused = true; - } finally { - pauseLock.unlock(); - } - } - - /** Resumes the Spider crawling. */ - public void resume() { - pauseLock.lock(); - try { - paused = false; - // Wake up all threads that are currently paused - pausedCondition.signalAll(); - } finally { - pauseLock.unlock(); - } - } - - /** - * Sets the spider so it will scan from the point of view of a user. - * - * @param user the user to be scanned as - */ - public void setScanAsUser(User user) { - this.scanUser = user; - } - - /** - * Gets the user that will be used in the scanning. - * - * @return the scan user - */ - protected User getScanUser() { - return this.scanUser; - } - - /** - * This method is run by each thread in the Thread Pool before the task execution. Particularly, - * it checks if the Spidering process is paused and, if it is, it waits on the corresponding - * condition for the process to be resumed. Called from the SpiderTask. - */ - protected void preTaskExecution() { - checkPauseAndWait(); - } - - /** - * This method is run by Threads in the ThreadPool and checks if the scan is paused and, if it - * is, waits until it's unpaused. - */ - protected void checkPauseAndWait() { - pauseLock.lock(); - try { - while (paused && !stopped) { - pausedCondition.await(); - } - } catch (InterruptedException e) { - } finally { - pauseLock.unlock(); - } - } - - /** - * This method is run by each thread in the Thread Pool after the task execution. Particularly, - * it notifies the listeners of the progress and checks if the scan is complete. Called from the - * SpiderTask. - */ - protected synchronized void postTaskExecution() { - if (stopped) { - // Stopped, so don't count the task(s) as done. - // (worker threads call this method even if the task was not really executed.) - return; - } - tasksDoneCount++; - int percentageComplete = tasksDoneCount * 100 / tasksTotalCount; - - // Compute the progress and notify the listeners - this.notifyListenersSpiderProgress( - percentageComplete, tasksDoneCount, tasksTotalCount - tasksDoneCount); - - // Check for ending conditions - if (tasksDoneCount == tasksTotalCount && initialized) { - this.complete(); - } - } - - /** - * Checks if is paused. - * - * @return true, if is paused - */ - public boolean isPaused() { - return this.paused; - } - - /** - * Checks if is stopped, i.e. a shutdown was issued or it is not running. - * - * @return true, if is stopped - */ - public boolean isStopped() { - if (!stopped && this.spiderParam.getMaxDuration() > 0) { - // Check to see if the scan has exceeded the specified maxDuration - if (TimeUnit.MILLISECONDS.toMinutes(System.currentTimeMillis() - this.timeStarted) - > this.spiderParam.getMaxDuration()) { - log.info( - "Spidering process has exceeded maxDuration of {} minute(s)", - this.spiderParam.getMaxDuration()); - this.complete(); - } - } - return stopped; - } - - /** - * Checks if is terminated. - * - * @return true, if is terminated - */ - public boolean isTerminated() { - return threadPool.isTerminated(); - } - - /* LISTENERS SECTION */ - - /** - * Adds a new spider listener. - * - * @param listener the listener - */ - public void addSpiderListener(SpiderListener listener) { - this.listeners.add(listener); - } - - /** - * Removes a spider listener. - * - * @param listener the listener - */ - public void removeSpiderListener(SpiderListener listener) { - this.listeners.remove(listener); - } - - /** - * Notifies all the listeners regarding the spider progress. - * - * @param percentageComplete the percentage complete - * @param numberCrawled the number of pages crawled - * @param numberToCrawl the number of pages left to crawl - */ - protected synchronized void notifyListenersSpiderProgress( - int percentageComplete, int numberCrawled, int numberToCrawl) { - for (SpiderListener l : listeners) { - l.spiderProgress(percentageComplete, numberCrawled, numberToCrawl); - } - } - - /** - * Notifies the listeners regarding a found uri. - * - * @param uri the uri - * @param method the method used for fetching the resource - * @param status the {@code FetchStatus} stating if this uri will be processed, and, if not, - * stating the reason of the filtering - */ - protected synchronized void notifyListenersFoundURI( - String uri, - String method, - org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus status) { - for (SpiderListener l : listeners) { - l.foundURI(uri, method, status); - } - } - - /** - * Notifies the listeners of a {@link SpiderTask}'s result. - * - * @param result the result of a spider task. - */ - protected synchronized void notifyListenersSpiderTaskResult(SpiderTaskResult result) { - for (SpiderListener l : listeners) { - l.notifySpiderTaskResult(result); - } - } - - /** - * Notifies the listeners that the spider is complete. - * - * @param successful {@code true} if the spider completed successfully (e.g. was not stopped), - * {@code false} otherwise - */ - protected synchronized void notifyListenersSpiderComplete(boolean successful) { - for (SpiderListener l : listeners) { - l.spiderComplete(successful); - } - } - - public void addCustomParser(org.zaproxy.zap.spider.parser.SpiderParser sp) { - this.controller.addSpiderParser(sp); - } - - private static class SpiderThreadFactory implements ThreadFactory { - - private final AtomicInteger threadNumber; - private final String namePrefix; - private final ThreadGroup group; - - public SpiderThreadFactory(String namePrefix) { - threadNumber = new AtomicInteger(1); - this.namePrefix = namePrefix; - group = Thread.currentThread().getThreadGroup(); - } - - @Override - public Thread newThread(Runnable r) { - Thread t = new Thread(group, r, namePrefix + threadNumber.getAndIncrement(), 0); - if (t.isDaemon()) { - t.setDaemon(false); - } - if (t.getPriority() != Thread.NORM_PRIORITY) { - t.setPriority(Thread.NORM_PRIORITY); - } - return t; - } - } -} diff --git a/zap/src/main/java/org/zaproxy/zap/spider/SpiderController.java b/zap/src/main/java/org/zaproxy/zap/spider/SpiderController.java deleted file mode 100644 index a4063f1e62b..00000000000 --- a/zap/src/main/java/org/zaproxy/zap/spider/SpiderController.java +++ /dev/null @@ -1,405 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2012 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.spider; - -import java.util.Collections; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; -import net.htmlparser.jericho.Config; -import org.apache.commons.httpclient.URI; -import org.apache.commons.httpclient.URIException; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.parosproxy.paros.network.HttpHeaderField; - -/** - * The SpiderController is used to manage the crawling process and interacts directly with the - * Spider Task threads. - * - * @deprecated (2.12.0) See the spider add-on in zap-extensions instead. - */ -@Deprecated -public class SpiderController implements org.zaproxy.zap.spider.parser.SpiderParserListener { - - /** The fetch filters used by the spider to filter the resources which are fetched. */ - private LinkedList fetchFilters; - - /** - * The parse filters used by the spider to filter the resources which were fetched, but should - * not be parsed. - */ - private LinkedList parseFilters; - - private org.zaproxy.zap.spider.filters.ParseFilter defaultParseFilter; - - /** The parsers used by the spider. */ - private LinkedList parsers; - - private List parsersUnmodifiableView; - - /** The spider. */ - private Spider spider; - - /** The resources visited as a set. */ - private Set visitedResources; - - /** The Constant log. */ - private static final Logger log = LogManager.getLogger(SpiderController.class); - - /** - * Instantiates a new spider controller. - * - * @param spider the spider - * @param customParsers the custom spider parsers - */ - protected SpiderController( - Spider spider, List customParsers) { - super(); - this.spider = spider; - this.fetchFilters = new LinkedList<>(); - this.parseFilters = new LinkedList<>(); - this.visitedResources = new HashSet<>(); - - prepareDefaultParsers(); - for (org.zaproxy.zap.spider.parser.SpiderParser parser : customParsers) { - this.addSpiderParser(parser); - } - } - - private void prepareDefaultParsers() { - this.parsers = new LinkedList<>(); - org.zaproxy.zap.spider.parser.SpiderParser parser; - - // If parsing of robots.txt is enabled - if (spider.getSpiderParam().isParseRobotsTxt()) { - parser = - new org.zaproxy.zap.spider.parser.SpiderRobotstxtParser( - spider.getSpiderParam()); - parsers.add(parser); - } - - // If parsing of sitemap.xml is enabled - if (spider.getSpiderParam().isParseSitemapXml()) { - log.debug("Adding SpiderSitemapXMLParser"); - parser = - new org.zaproxy.zap.spider.parser.SpiderSitemapXMLParser( - spider.getSpiderParam()); - parsers.add(parser); - } else { - log.debug("NOT Adding SpiderSitemapXMLParser"); - } - - // If parsing of SVN entries is enabled - if (spider.getSpiderParam().isParseSVNEntries()) { - parser = - new org.zaproxy.zap.spider.parser.SpiderSVNEntriesParser( - spider.getSpiderParam()); - parsers.add(parser); - } - - // If parsing of GIT entries is enabled - if (spider.getSpiderParam().isParseGit()) { - parser = new org.zaproxy.zap.spider.parser.SpiderGitParser(spider.getSpiderParam()); - parsers.add(parser); - } - - // Redirect requests parser - parser = new org.zaproxy.zap.spider.parser.SpiderRedirectParser(spider.getSpiderParam()); - parsers.add(parser); - - // HTTP Header parser - parser = new org.zaproxy.zap.spider.parser.SpiderHttpHeaderParser(spider.getSpiderParam()); - parsers.add(parser); - - // Simple HTML parser - parser = new org.zaproxy.zap.spider.parser.SpiderHtmlParser(spider.getSpiderParam()); - this.parsers.add(parser); - - // HTML Form parser - parser = - new org.zaproxy.zap.spider.parser.SpiderHtmlFormParser( - spider.getSpiderParam(), spider.getExtensionSpider().getValueGenerator()); - this.parsers.add(parser); - Config.CurrentCompatibilityMode.setFormFieldNameCaseInsensitive(false); - - // Prepare the parsers for OData ATOM files - parser = new org.zaproxy.zap.spider.parser.SpiderODataAtomParser(spider.getSpiderParam()); - this.parsers.add(parser); - - // Prepare the parsers for simple non-HTML files - parser = new org.zaproxy.zap.spider.parser.SpiderTextParser(spider.getSpiderParam()); - this.parsers.add(parser); - - this.parsersUnmodifiableView = Collections.unmodifiableList(parsers); - } - - /** - * Adds a new seed, if it wasn't already processed. - * - * @param uri the uri - * @param method the http method used for fetching the resource - */ - protected void addSeed(URI uri, String method) { - org.zaproxy.zap.spider.parser.SpiderResourceFound resourceFound = - org.zaproxy.zap.spider.parser.SpiderResourceFound.builder() - .setUri(uri.toString()) - .setMethod(method) - .build(); - // Check if the uri was processed already - String resourceIdentifier = ""; - try { - resourceIdentifier = buildCanonicalResourceIdentifier(uri, resourceFound); - } catch (URIException e) { - return; - } - synchronized (visitedResources) { - if (visitedResources.contains(resourceIdentifier)) { - log.debug("URI already visited: {}", uri); - return; - } else { - visitedResources.add(resourceIdentifier); - } - } - // Create and submit the new task - SpiderTask task = new SpiderTask(spider, resourceFound, uri); - spider.submitTask(task); - // Add the uri to the found list - spider.notifyListenersFoundURI( - uri.toString(), - method, - org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus.SEED); - } - - /** - * Gets the fetch filters used by the spider during the spidering process. - * - * @return the fetch filters - */ - protected LinkedList getFetchFilters() { - return fetchFilters; - } - - /** - * Adds a new fetch filter to the spider. - * - * @param filter the filter - */ - public void addFetchFilter(org.zaproxy.zap.spider.filters.FetchFilter filter) { - log.debug("Loading fetch filter: {}", filter.getClass().getSimpleName()); - fetchFilters.add(filter); - } - - /** - * Gets the parses the filters. - * - * @return the parses the filters - */ - protected LinkedList getParseFilters() { - return parseFilters; - } - - /** - * Adds the parse filter to the spider controller. - * - * @param filter the filter - */ - public void addParseFilter(org.zaproxy.zap.spider.filters.ParseFilter filter) { - log.debug("Loading parse filter: {}", filter.getClass().getSimpleName()); - parseFilters.add(filter); - } - - protected void setDefaultParseFilter(org.zaproxy.zap.spider.filters.ParseFilter filter) { - log.debug("Setting Default filter: {}", filter.getClass().getSimpleName()); - defaultParseFilter = filter; - } - - protected org.zaproxy.zap.spider.filters.ParseFilter getDefaultParseFilter() { - return defaultParseFilter; - } - - public void init() { - visitedResources.clear(); - - for (org.zaproxy.zap.spider.parser.SpiderParser parser : parsers) { - parser.addSpiderParserListener(this); - } - } - - /** Clears the previous process. */ - public void reset() { - visitedResources.clear(); - - for (org.zaproxy.zap.spider.parser.SpiderParser parser : parsers) { - parser.removeSpiderParserListener(this); - } - } - - /** - * Builds a canonical identifier for found resources considering the method, URI, headers, and - * body. - * - * @param uri uniform resource identifier for resource - * @param resourceFound resource found - * @return identifier as a string representation usable for equality checks - */ - private String buildCanonicalResourceIdentifier( - URI uri, org.zaproxy.zap.spider.parser.SpiderResourceFound resourceFound) - throws URIException { - StringBuilder identifierBuilder = new StringBuilder(50); - String visitedURI = - URLCanonicalizer.buildCleanedParametersURIRepresentation( - uri, - spider.getSpiderParam().getHandleParameters(), - spider.getSpiderParam().isHandleODataParametersVisited(), - spider.getSpiderParam()::isIrrelevantUrlParameter); - identifierBuilder.append(resourceFound.getMethod()); - identifierBuilder.append(" "); - identifierBuilder.append(visitedURI); - identifierBuilder.append("\n"); - identifierBuilder.append(getCanonicalHeadersString(resourceFound.getHeaders())); - identifierBuilder.append("\n"); - identifierBuilder.append(resourceFound.getBody()); - return identifierBuilder.toString(); - } - - @Override - public void resourceFound(org.zaproxy.zap.spider.parser.SpiderResourceFound resourceFound) { - log.debug("New {} resource found: {}", resourceFound.getMethod(), resourceFound.getUri()); - - // Create the uri - URI uriV = createURI(resourceFound.getUri()); - if (uriV == null) { - return; - } - - // Check if the resource was processed already - String resourceIdentifier = ""; - try { - resourceIdentifier = buildCanonicalResourceIdentifier(uriV, resourceFound); - } catch (URIException e) { - return; - } - synchronized (visitedResources) { - if (visitedResources.contains(resourceIdentifier)) { - log.debug("Resource already visited: {}", resourceIdentifier.trim()); - return; - } else { - visitedResources.add(resourceIdentifier); - } - } - - // Check if any of the filters disallows this uri - for (org.zaproxy.zap.spider.filters.FetchFilter f : fetchFilters) { - org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus s = f.checkFilter(uriV); - if (s != org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus.VALID) { - log.debug("URI: {} was filtered by a filter with reason: {}", uriV, s); - spider.notifyListenersFoundURI( - resourceFound.getUri(), resourceFound.getMethod(), s); - return; - } - } - - // Check if resource should be ignored and not fetched - if (resourceFound.isShouldIgnore()) { - log.debug("URI: {} is valid, but will not be fetched, by parser recommendation.", uriV); - spider.notifyListenersFoundURI( - resourceFound.getUri(), - resourceFound.getMethod(), - org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus.VALID); - return; - } - - spider.notifyListenersFoundURI( - resourceFound.getUri(), - resourceFound.getMethod(), - org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus.VALID); - - // Submit the task - SpiderTask task = new SpiderTask(spider, resourceFound, uriV); - spider.submitTask(task); - } - - /** - * Builds a canonical string representation for HTTP header fields by sorting the headers based - * on the name, trimming and lowercasing the name and value, and removing duplicates. - * - * @param headers list of HTTP headers - * @return canonical string representation of headers - */ - private String getCanonicalHeadersString(List headers) { - return headers.stream() - .sorted((h1, h2) -> h1.getName().compareTo(h2.getName())) - .map( - h -> - h.getName().trim().toLowerCase() - + "=" - + h.getValue().trim().toLowerCase()) - .distinct() - .collect(Collectors.joining("|")); - } - - /** - * Creates the {@link URI} starting from the uri string. First it tries to convert it into a - * String considering it's already encoded and, if it fails, tries to create it considering it's - * not encoded. - * - * @param uri the string of the uri - * @return the URI, or null if an error occurred and the URI could not be constructed. - */ - private URI createURI(String uri) { - URI uriV = null; - try { - // Try to see if we can create the URI, considering it's encoded. - uriV = new URI(uri, true); - } catch (URIException e) { - // An error occurred, so try to create the URI considering it's not encoded. - try { - log.debug("Second try..."); - uriV = new URI(uri, false); - } catch (Exception ex) { - log.error("Error while converting to uri: {}", uri, ex); - return null; - } - // A non URIException occurred, so just ignore the URI - } catch (Exception e) { - log.error("Error while converting to uri: {}", uri, e); - return null; - } - return uriV; - } - - /** - * Gets an unmodifiable view of the list of that should be used during the scan. - * - * @return the parsers - */ - public List getParsers() { - return parsersUnmodifiableView; - } - - public void addSpiderParser(org.zaproxy.zap.spider.parser.SpiderParser parser) { - log.debug("Loading custom Spider Parser: {}", parser.getClass().getSimpleName()); - parser.setSpiderParam(spider.getSpiderParam()); - this.parsers.addFirst(parser); - } -} diff --git a/zap/src/main/java/org/zaproxy/zap/spider/SpiderListener.java b/zap/src/main/java/org/zaproxy/zap/spider/SpiderListener.java deleted file mode 100644 index b5dc5913593..00000000000 --- a/zap/src/main/java/org/zaproxy/zap/spider/SpiderListener.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2012 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.spider; - -/** - * The listener interface for receiving spider related events. The class that is interested in - * processing a spider event implements this interface, and the object created with that class is - * registered with a component using the component's addSpiderListener method. When the - * spider event occurs, that object's appropriate method is invoked. - * - * @deprecated (2.12.0) See the spider add-on in zap-extensions instead. - */ -@Deprecated -public interface SpiderListener { - - /** - * Event triggered when the Spider progress has changed. - * - * @param percentageComplete the percentage complete - * @param numberCrawled the number of pages crawled - * @param numberToCrawl the number of pages left to crawl - */ - void spiderProgress(int percentageComplete, int numberCrawled, int numberToCrawl); - - /** - * Event triggered when a new uri was found. The status parameter says if the URI - * was skipped according to any skip rule or it was processed. - * - * @param uri the uri - * @param method the method used for accessing the uri - * @param status the {@code FetchStatus} stating if this uri will be processed, and, if not, - * stating the reason of the filtering - */ - void foundURI( - String uri, - String method, - org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus status); - - /** - * Notifies that a new {@link SpiderTask}'s result is available. - * - * @param spiderTaskResult the result of the spider task. - */ - void notifySpiderTaskResult(SpiderTaskResult spiderTaskResult); - - /** - * Event triggered when the spider is finished. This event is triggered either when the spider - * has completed scanning a website, in which case the parameter successful is - * true , either when it was stopped by an user in which case the successful - * parameter is false. - * - * @param successful - *

    - *
  • true - when the spider has scanned a website completely and finished by its own - *
  • false - when the spider was stopped by the user - *
- */ - void spiderComplete(boolean successful); -} diff --git a/zap/src/main/java/org/zaproxy/zap/spider/SpiderParam.java b/zap/src/main/java/org/zaproxy/zap/spider/SpiderParam.java deleted file mode 100644 index 4e37f2ccf70..00000000000 --- a/zap/src/main/java/org/zaproxy/zap/spider/SpiderParam.java +++ /dev/null @@ -1,1066 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2012 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.spider; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Set; -import java.util.regex.Pattern; -import java.util.stream.Collectors; -import org.apache.commons.configuration.ConversionException; -import org.apache.commons.configuration.HierarchicalConfiguration; -import org.apache.commons.httpclient.URI; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.parosproxy.paros.Constant; -import org.parosproxy.paros.common.AbstractParam; -import org.parosproxy.paros.control.Control; -import org.zaproxy.zap.extension.api.ZapApiIgnore; -import org.zaproxy.zap.extension.httpsessions.ExtensionHttpSessions; - -/** - * The SpiderParam wraps all the parameters that are given to the spider. - * - * @deprecated (2.12.0) See the spider add-on in zap-extensions instead. - */ -@Deprecated -public class SpiderParam extends AbstractParam { - - /** - * The value that indicates that the crawl depth is unlimited. - * - * @since 2.8.0 - * @see #setMaxDepth(int) - */ - public static final int UNLIMITED_DEPTH = 0; - - /** The Constant SPIDER_MAX_DEPTH. */ - private static final String SPIDER_MAX_DEPTH = "spider.maxDepth"; - - /** The Constant SPIDER_THREAD. */ - private static final String SPIDER_THREAD = "spider.thread"; - - /** The Constant SPIDER_POST_FORM. */ - private static final String SPIDER_POST_FORM = "spider.postform"; - - /** The Constant SPIDER_PROCESS_FORM. */ - private static final String SPIDER_PROCESS_FORM = "spider.processform"; - - /** The Constant SPIDER_SKIP_URL. */ - private static final String SPIDER_SKIP_URL = "spider.skipurl"; - - /** The Constant SPIDER_REQUEST_WAIT. */ - private static final String SPIDER_REQUEST_WAIT = "spider.requestwait"; - - /** The Constant SPIDER_PARSE_COMMENTS. */ - private static final String SPIDER_PARSE_COMMENTS = "spider.parseComments"; - - /** The Constant SPIDER_PARSE_ROBOTS_TXT. */ - private static final String SPIDER_PARSE_ROBOTS_TXT = "spider.parseRobotsTxt"; - - /** The Constant SPIDER_PARSE_SITEMAP_XML. */ - private static final String SPIDER_PARSE_SITEMAP_XML = "spider.parseSitemapXml"; - - /** The Constant SPIDER_PARSE_SVN_ENTRIES. */ - private static final String SPIDER_PARSE_SVN_ENTRIES = "spider.parseSVNentries"; - - /** The Constant SPIDER_PARSE_GIT. */ - private static final String SPIDER_PARSE_GIT = "spider.parseGit"; - - /** The Constant SPIDER_HANDLE_PARAMETERS. */ - private static final String SPIDER_HANDLE_PARAMETERS = "spider.handleParameters"; - - /** The Constant SPIDER_HANDLE_ODATA_PARAMETERS. */ - private static final String SPIDER_HANDLE_ODATA_PARAMETERS = "spider.handleODataParameters"; - - private static final String DOMAIN_ALWAYS_IN_SCOPE_KEY = "spider.domainsAlwaysInScope"; - private static final String ALL_DOMAINS_ALWAYS_IN_SCOPE_KEY = - DOMAIN_ALWAYS_IN_SCOPE_KEY + ".domainAlwaysInScope"; - private static final String DOMAIN_ALWAYS_IN_SCOPE_VALUE_KEY = "name"; - private static final String DOMAIN_ALWAYS_IN_SCOPE_REGEX_KEY = "regex"; - private static final String DOMAIN_ALWAYS_IN_SCOPE_ENABLED_KEY = "enabled"; - private static final String CONFIRM_REMOVE_DOMAIN_ALWAYS_IN_SCOPE = - "spider.confirmRemoveDomainAlwaysInScope"; - - private static final String MAX_SCANS_IN_UI = "spider.maxScansInUI"; - - private static final String SHOW_ADV_DIALOG = "spider.advDialog"; - private static final String MAX_DURATION = "spider.maxDuration"; - - private static final String MAX_CHILDREN = "spider.maxChildren"; - - /** - * Configuration key to write/read the {@code sendRefererHeader} flag. - * - * @since 2.4.0 - * @see #sendRefererHeader - */ - private static final String SPIDER_SENDER_REFERER_HEADER = "spider.sendRefererHeader"; - - /** Configuration key to write/read the {@link #acceptCookies} flag. */ - private static final String SPIDER_ACCEPT_COOKIES = "spider.acceptCookies"; - - /** Configuration key to write/read the {@link #maxParseSizeBytes} flag. */ - private static final String SPIDER_MAX_PARSE_SIZE_BYTES = "spider.maxParseSizeBytes"; - - /** - * Default maximum size, in bytes, that a response might have to be parsed. - * - * @see #maxParseSizeBytes - */ - private static final int DEFAULT_MAX_PARSE_SIZE_BYTES = 2621440; // 2.5 MiB - - /** Configuration key to write/read the {@link #irrelevantUrlParameters} property. */ - private static final String SPIDER_IRRELEVANT_URL_PARAMETERS = "spider.irrelevantUrlParameters"; - - private static ExtensionHttpSessions extensionHttpSessions; - - /** - * This option is used to define how the parameters are used when checking if an URI was already - * visited. - */ - public enum HandleParametersOption { - /** The parameters are ignored completely. */ - IGNORE_COMPLETELY, - /** Only the name of the parameter is used, but the value is ignored. */ - IGNORE_VALUE, - /** Both the name and value of the parameter are used. */ - USE_ALL; - - public String getName() { - switch (this) { - case IGNORE_COMPLETELY: - return Constant.messages.getString( - "spider.options.value.handleparameters.ignoreAll"); - case IGNORE_VALUE: - return Constant.messages.getString( - "spider.options.value.handleparameters.ignoreValue"); - case USE_ALL: - return Constant.messages.getString( - "spider.options.value.handleparameters.useAll"); - default: - return null; - } - } - } - - /** The max depth of the crawling. */ - private int maxDepth = 5; - - /** The thread count. */ - private int threadCount = 2; - - /** Whether comments should be parsed for URIs. */ - private boolean parseComments = true; - - /** Whether robots.txt file should be parsed for URIs. */ - private boolean parseRobotsTxt = true; - - /** Whether sitemap.xml file should be parsed for URIs. */ - private boolean parseSitemapXml = true; - - /** Whether SVN entries files should be parsed for URIs. */ - private boolean parseSVNentries = false; - - /** Whether Git files should be parsed for URIs. */ - private boolean parseGit = false; - - /** Whether the forms are processed and submitted at all. */ - private boolean processForm = true; - - /** - * Whether the forms are submitted, if their method is HTTP POST. This option should not be used - * if the forms are not processed at all (processForm). - */ - private boolean postForm = true; - - /** The waiting time between new requests to server - safe from DDOS. */ - private int requestWait = 200; - - /** Which urls are skipped. */ - private String skipURL = ""; - - /** The pattern for skip url. */ - private Pattern patternSkipURL = null; - - /** The user agent string, if different than the default one. */ - private String userAgent = null; - - /** The handle parameters visited. */ - private HandleParametersOption handleParametersVisited = HandleParametersOption.USE_ALL; - - /** - * Defines if we take care of OData specific parameters during the visit in order to identify - * known URL * - */ - private boolean handleODataParametersVisited = false; - - /** The maximum duration in minutes that the spider is allowed to run for, 0 meaning no limit */ - private int maxDuration = 0; - - /** The maximum number of child nodes (per node) that can be crawled, 0 means no limit. */ - private int maxChildren; - - private List domainsAlwaysInScope = new ArrayList<>(0); - private List domainsAlwaysInScopeEnabled = new ArrayList<>(0); - private boolean confirmRemoveDomainAlwaysInScope; - private int maxScansInUI = 5; - private boolean showAdvancedDialog = false; // TODO load/save - - /** The log. */ - private static final Logger log = LogManager.getLogger(SpiderParam.class); - - /** - * Flag that indicates if the 'Referer' header should be sent while spidering. - * - *

Default value is {@code true}. - * - * @since 2.4.0 - * @see #SPIDER_SENDER_REFERER_HEADER - * @see #isSendRefererHeader() - * @see #setSendRefererHeader(boolean) - */ - private boolean sendRefererHeader = true; - - /** - * Flag that indicates if a spider process should accept cookies. - * - *

Default value is {@code true}. - * - * @see #SPIDER_ACCEPT_COOKIES - * @see #isAcceptCookies() - * @see #setAcceptCookies(boolean) - */ - private boolean acceptCookies = true; - - /** - * The maximum size, in bytes, that a response might have to be parsed. - * - *

Default value is {@value #DEFAULT_MAX_PARSE_SIZE_BYTES} bytes. - * - * @see #SPIDER_MAX_PARSE_SIZE_BYTES - * @see #getMaxParseSizeBytes() - * @see #setMaxParseSizeBytes(int) - */ - private int maxParseSizeBytes = DEFAULT_MAX_PARSE_SIZE_BYTES; - - /** - * Parameter names which are ignored in the {@link URLCanonicalizer}. - * - * @see #SPIDER_IRRELEVANT_URL_PARAMETERS - */ - private Set irrelevantUrlParameters = Collections.emptySet(); - - /** Instantiates a new spider param. */ - public SpiderParam() {} - - @Override - protected void parse() { - updateOptions(); - - extensionHttpSessions = - Control.getSingleton() - .getExtensionLoader() - .getExtension(ExtensionHttpSessions.class); - - this.threadCount = getInt(SPIDER_THREAD, 2); - - this.maxDepth = getInt(SPIDER_MAX_DEPTH, 5); - - this.maxDuration = getInt(MAX_DURATION, 0); - - this.maxChildren = getInt(MAX_CHILDREN, 0); - - this.maxScansInUI = getInt(MAX_SCANS_IN_UI, 5); - - this.showAdvancedDialog = getBoolean(SHOW_ADV_DIALOG, false); - - this.processForm = getBoolean(SPIDER_PROCESS_FORM, true); - - try { - this.postForm = getConfig().getBoolean(SPIDER_POST_FORM, true); - } catch (ConversionException e) { - // conversion issue from 1.4.1: convert the field from int to boolean - log.info( - "Warning while parsing config file: " - + SPIDER_POST_FORM - + " was not in the expected format due to an upgrade. Converting it!"); - this.postForm = !getConfig().getProperty(SPIDER_POST_FORM).toString().equals("0"); - getConfig().setProperty(SPIDER_POST_FORM, String.valueOf(postForm)); - } - - this.requestWait = getInt(SPIDER_REQUEST_WAIT, 200); - - this.parseComments = getBoolean(SPIDER_PARSE_COMMENTS, true); - - this.parseRobotsTxt = getBoolean(SPIDER_PARSE_ROBOTS_TXT, true); - - this.parseSitemapXml = getBoolean(SPIDER_PARSE_SITEMAP_XML, true); - - this.parseSVNentries = getBoolean(SPIDER_PARSE_SVN_ENTRIES, false); - - this.parseGit = getBoolean(SPIDER_PARSE_GIT, false); - - this.skipURL = getString(SPIDER_SKIP_URL, ""); - parseSkipURL(this.skipURL); - - handleParametersVisited = - HandleParametersOption.valueOf( - getString( - SPIDER_HANDLE_PARAMETERS, - HandleParametersOption.USE_ALL.toString())); - - this.handleODataParametersVisited = getBoolean(SPIDER_HANDLE_ODATA_PARAMETERS, false); - - loadDomainsAlwaysInScope(); - this.confirmRemoveDomainAlwaysInScope = - getBoolean(CONFIRM_REMOVE_DOMAIN_ALWAYS_IN_SCOPE, true); - - this.sendRefererHeader = getBoolean(SPIDER_SENDER_REFERER_HEADER, true); - - this.acceptCookies = getBoolean(SPIDER_ACCEPT_COOKIES, true); - - this.maxParseSizeBytes = getInt(SPIDER_MAX_PARSE_SIZE_BYTES, DEFAULT_MAX_PARSE_SIZE_BYTES); - - this.irrelevantUrlParameters = - getStringSet(SPIDER_IRRELEVANT_URL_PARAMETERS, this.irrelevantUrlParameters); - } - - private void updateOptions() { - final String oldDomainsInScope = "spider.scope"; - if (getConfig().containsKey(oldDomainsInScope)) { - migrateOldDomainsInScopeOption(getConfig().getString(oldDomainsInScope, "")); - getConfig().clearProperty(oldDomainsInScope); - } - } - - private void migrateOldDomainsInScopeOption(String oldDomainsInScope) { - List domainsInScope = - convertOldDomainsInScopeOption(oldDomainsInScope); - - if (!domainsInScope.isEmpty()) { - setDomainsAlwaysInScope(domainsInScope); - } - } - - private static List convertOldDomainsInScopeOption( - String oldDomainsInScope) { - if (oldDomainsInScope == null || oldDomainsInScope.isEmpty()) { - return Collections.emptyList(); - } - - ArrayList domainsInScope = new ArrayList<>(); - String[] names = oldDomainsInScope.split(";"); - for (String name : names) { - String domain = name.trim(); - if (!domain.isEmpty()) { - if (domain.contains("*")) { - domain = domain.replace(".", "\\.").replace("+", "\\+").replace("*", ".*?"); - try { - Pattern pattern = Pattern.compile(domain, Pattern.CASE_INSENSITIVE); - domainsInScope.add(new DomainAlwaysInScopeMatcher(pattern)); - } catch (IllegalArgumentException e) { - log.error("Failed to migrate a domain always in scope, name: {}", name, e); - } - } else { - domainsInScope.add(new DomainAlwaysInScopeMatcher(domain)); - } - } - } - domainsInScope.trimToSize(); - return domainsInScope; - } - - /** - * Gets the maximum depth the spider can crawl. - * - * @return the maximum depth, or {@value #UNLIMITED_DEPTH} if unlimited. - * @see #setMaxDepth(int) - */ - public int getMaxDepth() { - return maxDepth; - } - - /** - * Sets the maximum depth the spider can crawl. - * - *

Value {@value #UNLIMITED_DEPTH} for unlimited depth. - * - * @param maxDepth the new maximum depth. - * @see #getMaxDepth() - */ - public void setMaxDepth(int maxDepth) { - this.maxDepth = maxDepth > UNLIMITED_DEPTH ? maxDepth : UNLIMITED_DEPTH; - getConfig().setProperty(SPIDER_MAX_DEPTH, Integer.toString(this.maxDepth)); - } - - /** - * Gets the thread count. - * - * @return Returns the thread count - */ - public int getThreadCount() { - return threadCount; - } - - /** - * Sets the thread count. - * - * @param thread The thread count to set. - */ - public void setThreadCount(int thread) { - this.threadCount = thread; - getConfig().setProperty(SPIDER_THREAD, Integer.toString(this.threadCount)); - } - - /** - * Checks if is the forms should be submitted with the HTTP POST method. This option should not - * be used if the forms are not processed at all (processForm). - * - * @return true, if the forms should be posted. - */ - public boolean isPostForm() { - return postForm; - } - - /** - * Sets if the forms should be submitted with the HTTP POST method. This option should not be - * used if the forms are not processed at all (processForm). - * - * @param postForm the new post form status - */ - public void setPostForm(boolean postForm) { - this.postForm = postForm; - getConfig().setProperty(SPIDER_POST_FORM, Boolean.toString(postForm)); - } - - /** - * Checks if the forms should be processed. - * - * @return true, if the forms should be processed - */ - public boolean isProcessForm() { - return processForm; - } - - /** - * Sets if the forms should be processed. - * - * @param processForm the new process form status - */ - public void setProcessForm(boolean processForm) { - this.processForm = processForm; - getConfig().setProperty(SPIDER_PROCESS_FORM, Boolean.toString(processForm)); - } - - /** - * Sets the skip url string. This string is being parsed into a pattern which is used to check - * if a url should be skipped while crawling. - * - * @param skipURL the new skip url string - */ - public void setSkipURLString(String skipURL) { - this.skipURL = skipURL; - getConfig().setProperty(SPIDER_SKIP_URL, this.skipURL); - parseSkipURL(this.skipURL); - } - - /** - * Gets the skip url string. - * - * @return the skip url string - */ - public String getSkipURLString() { - return skipURL; - } - - /** - * Checks if is this url should be skipped. - * - * @param uri the uri - * @return true, if the url should be skipped - */ - public boolean isSkipURL(URI uri) { - if (patternSkipURL == null || uri == null) { - return false; - } - String sURI = uri.toString(); - return patternSkipURL.matcher(sURI).find(); - } - - /** - * Parses the skip url string. - * - * @param skipURL the skip url string - */ - private void parseSkipURL(String skipURL) { - patternSkipURL = null; - - if (skipURL == null || skipURL.equals("")) { - return; - } - - skipURL = skipURL.replaceAll("\\.", "\\\\."); - skipURL = skipURL.replaceAll("\\*", ".*?").replaceAll("(\\s+$)|(^\\s+)", ""); - skipURL = "\\A(" + skipURL.replaceAll("\\s+", "|") + ")"; - patternSkipURL = Pattern.compile(skipURL, Pattern.CASE_INSENSITIVE | Pattern.MULTILINE); - } - - /** - * Gets the time between the requests sent to a server. - * - * @return the request wait time - */ - public int getRequestWaitTime() { - return requestWait; - } - - /** - * Sets the time between the requests sent to a server. - * - * @param requestWait the new request wait time - */ - public void setRequestWaitTime(int requestWait) { - this.requestWait = requestWait; - this.getConfig().setProperty(SPIDER_REQUEST_WAIT, Integer.toString(requestWait)); - } - - /** - * Gets the user agent. - * - * @return the user agent - */ - public String getUserAgent() { - return userAgent; - } - - /** - * Sets the user agent, if different from the default one. - * - * @param userAgent the new user agent - */ - public void setUserAgent(String userAgent) { - this.userAgent = userAgent; - } - - /** - * Checks if the spider should parse the comments. - * - * @return true, if it parses the comments - */ - public boolean isParseComments() { - return parseComments; - } - - /** - * Sets the whether the spider parses the comments. - * - * @param parseComments the new parses the comments value - */ - public void setParseComments(boolean parseComments) { - this.parseComments = parseComments; - getConfig().setProperty(SPIDER_PARSE_COMMENTS, Boolean.toString(parseComments)); - } - - /** - * Checks if the spider should parse the robots.txt for uris (not related to following the - * directions). - * - * @return true, if it parses the file - */ - public boolean isParseRobotsTxt() { - return parseRobotsTxt; - } - - /** - * Checks if the spider should parse the sitemap.xml for URIs. - * - * @return true, if it parses the file - */ - public boolean isParseSitemapXml() { - return parseSitemapXml; - } - - /** - * Checks if the spider should parse the SVN entries files for URIs (not related to following - * the directions). - * - * @return true, if it parses the file - */ - public boolean isParseSVNEntries() { - return parseSVNentries; - } - - /** - * Checks if the spider should parse the Git files for URIs. - * - * @return true, if it parses the files - */ - public boolean isParseGit() { - return parseGit; - } - - /** - * Sets the whether the spider parses the robots.txt for uris (not related to following the - * directions). - * - * @param parseRobotsTxt the new value for parseRobotsTxt - */ - public void setParseRobotsTxt(boolean parseRobotsTxt) { - this.parseRobotsTxt = parseRobotsTxt; - getConfig().setProperty(SPIDER_PARSE_ROBOTS_TXT, Boolean.toString(parseRobotsTxt)); - } - - /** - * Sets the whether the spider parses the sitemap.xml for URIs. - * - * @param parseSitemapXml the new value for parseSitemapXml - */ - public void setParseSitemapXml(boolean parseSitemapXml) { - this.parseSitemapXml = parseSitemapXml; - getConfig().setProperty(SPIDER_PARSE_SITEMAP_XML, Boolean.toString(parseSitemapXml)); - } - - /** - * Sets the whether the spider parses the SVN entries file for URIs (not related to following - * the directions). - * - * @param parseSVNentries the new value for parseSVNentries - */ - public void setParseSVNEntries(boolean parseSVNentries) { - this.parseSVNentries = parseSVNentries; - getConfig().setProperty(SPIDER_PARSE_SVN_ENTRIES, Boolean.toString(parseSVNentries)); - } - - /** - * Sets the whether the spider parses Git files for URIs - * - * @param parseGit the new value for parseGit - */ - public void setParseGit(boolean parseGit) { - this.parseGit = parseGit; - getConfig().setProperty(SPIDER_PARSE_GIT, Boolean.toString(parseGit)); - } - - /** - * Gets how the spider handles parameters when checking URIs visited. - * - * @return the handle parameters visited - */ - public HandleParametersOption getHandleParameters() { - return handleParametersVisited; - } - - /** - * Sets the how the spider handles parameters when checking URIs visited. - * - * @param handleParametersVisited the new handle parameters visited value - */ - public void setHandleParameters(HandleParametersOption handleParametersVisited) { - this.handleParametersVisited = handleParametersVisited; - getConfig().setProperty(SPIDER_HANDLE_PARAMETERS, handleParametersVisited.toString()); - } - - /** - * Sets the how the spider handles parameters when checking URIs visited. - * - *

The provided parameter is, in this case, a String which is cast to the proper value. - * Possible values are: {@code "USE_ALL"}, {@code "IGNORE_VALUE"}, {@code "IGNORE_COMPLETELY"}. - * - * @param handleParametersVisited the new handle parameters visited value - * @throws IllegalArgumentException if the given parameter is not a value of {@code - * HandleParametersOption}. - */ - public void setHandleParameters(String handleParametersVisited) { - this.handleParametersVisited = HandleParametersOption.valueOf(handleParametersVisited); - getConfig().setProperty(SPIDER_HANDLE_PARAMETERS, this.handleParametersVisited.toString()); - } - - /** - * Check if the spider should take into account OData-specific parameters (i.e : resource - * identifiers) in order to identify already visited URL - * - * @return true, for handling OData parameters - */ - public boolean isHandleODataParametersVisited() { - return handleODataParametersVisited; - } - - /** - * Defines if the spider should handle OData specific parameters (i.e : resource identifiers) To - * identify already visited URL - * - * @param handleODataParametersVisited the new value for handleODataParametersVisited - */ - public void setHandleODataParametersVisited(boolean handleODataParametersVisited) { - this.handleODataParametersVisited = handleODataParametersVisited; - getConfig() - .setProperty( - SPIDER_HANDLE_ODATA_PARAMETERS, - Boolean.toString(handleODataParametersVisited)); - } - - /** - * Returns the domains that will be always in scope. - * - * @return the domains that will be always in scope. - * @since 2.3.0 - * @see #getDomainsAlwaysInScopeEnabled() - * @see #setDomainsAlwaysInScope(List) - */ - @ZapApiIgnore - public List getDomainsAlwaysInScope() { - return domainsAlwaysInScope; - } - - /** - * Returns the, enabled, domains that will be always in scope. - * - * @return the enabled domains that will be always in scope. - * @since 2.3.0 - * @see #getDomainsAlwaysInScope() - * @see #setDomainsAlwaysInScope(List) - */ - @ZapApiIgnore - public List getDomainsAlwaysInScopeEnabled() { - return domainsAlwaysInScopeEnabled; - } - - /** - * Sets the domains that will be always in scope. - * - * @param domainsAlwaysInScope the domains that will be excluded. - * @since 2.3.0 - */ - public void setDomainsAlwaysInScope(List domainsAlwaysInScope) { - if (domainsAlwaysInScope == null || domainsAlwaysInScope.isEmpty()) { - ((HierarchicalConfiguration) getConfig()).clearTree(ALL_DOMAINS_ALWAYS_IN_SCOPE_KEY); - - this.domainsAlwaysInScope = Collections.emptyList(); - this.domainsAlwaysInScopeEnabled = Collections.emptyList(); - return; - } - - this.domainsAlwaysInScope = new ArrayList<>(domainsAlwaysInScope); - - ((HierarchicalConfiguration) getConfig()).clearTree(ALL_DOMAINS_ALWAYS_IN_SCOPE_KEY); - - int size = domainsAlwaysInScope.size(); - ArrayList enabledExcludedDomains = new ArrayList<>(size); - for (int i = 0; i < size; ++i) { - String elementBaseKey = ALL_DOMAINS_ALWAYS_IN_SCOPE_KEY + "(" + i + ")."; - DomainAlwaysInScopeMatcher excludedDomain = domainsAlwaysInScope.get(i); - - getConfig() - .setProperty( - elementBaseKey + DOMAIN_ALWAYS_IN_SCOPE_VALUE_KEY, - excludedDomain.getValue()); - getConfig() - .setProperty( - elementBaseKey + DOMAIN_ALWAYS_IN_SCOPE_REGEX_KEY, - excludedDomain.isRegex()); - getConfig() - .setProperty( - elementBaseKey + DOMAIN_ALWAYS_IN_SCOPE_ENABLED_KEY, - excludedDomain.isEnabled()); - - if (excludedDomain.isEnabled()) { - enabledExcludedDomains.add(excludedDomain); - } - } - - enabledExcludedDomains.trimToSize(); - this.domainsAlwaysInScopeEnabled = enabledExcludedDomains; - } - - private void loadDomainsAlwaysInScope() { - List fields = - ((HierarchicalConfiguration) getConfig()) - .configurationsAt(ALL_DOMAINS_ALWAYS_IN_SCOPE_KEY); - this.domainsAlwaysInScope = new ArrayList<>(fields.size()); - ArrayList domainsInScopeEnabled = - new ArrayList<>(fields.size()); - for (HierarchicalConfiguration sub : fields) { - String value = sub.getString(DOMAIN_ALWAYS_IN_SCOPE_VALUE_KEY, ""); - if ("".equals(value)) { - log.warn( - "Failed to read an spider domain in scope entry, required value is empty."); - } - - DomainAlwaysInScopeMatcher excludedDomain = null; - boolean regex = sub.getBoolean(DOMAIN_ALWAYS_IN_SCOPE_REGEX_KEY, false); - if (regex) { - try { - Pattern pattern = DomainAlwaysInScopeMatcher.createPattern(value); - excludedDomain = new DomainAlwaysInScopeMatcher(pattern); - } catch (IllegalArgumentException e) { - log.error( - "Failed to read an spider domain in scope entry with regex: {}", - value, - e); - } - } else { - excludedDomain = new DomainAlwaysInScopeMatcher(value); - } - - if (excludedDomain != null) { - excludedDomain.setEnabled(sub.getBoolean(DOMAIN_ALWAYS_IN_SCOPE_ENABLED_KEY, true)); - - domainsAlwaysInScope.add(excludedDomain); - - if (excludedDomain.isEnabled()) { - domainsInScopeEnabled.add(excludedDomain); - } - } - } - - domainsInScopeEnabled.trimToSize(); - this.domainsAlwaysInScopeEnabled = domainsInScopeEnabled; - } - - /** - * Tells whether or not the remotion of a "domain always in scope" needs confirmation. - * - * @return {@code true} if the remotion needs confirmation, {@code false} otherwise. - * @since 2.3.0 - */ - @ZapApiIgnore - public boolean isConfirmRemoveDomainAlwaysInScope() { - return this.confirmRemoveDomainAlwaysInScope; - } - - /** - * Sets whether or not the remotion of a "domain always in scope" needs confirmation. - * - * @param confirmRemove {@code true} if the remotion needs confirmation, {@code false} - * otherwise. - * @since 2.3.0 - */ - @ZapApiIgnore - public void setConfirmRemoveDomainAlwaysInScope(boolean confirmRemove) { - this.confirmRemoveDomainAlwaysInScope = confirmRemove; - getConfig() - .setProperty( - CONFIRM_REMOVE_DOMAIN_ALWAYS_IN_SCOPE, confirmRemoveDomainAlwaysInScope); - } - - public int getMaxScansInUI() { - return maxScansInUI; - } - - public void setMaxScansInUI(int maxScansInUI) { - this.maxScansInUI = maxScansInUI; - getConfig().setProperty(MAX_SCANS_IN_UI, this.maxScansInUI); - } - - public boolean isShowAdvancedDialog() { - return showAdvancedDialog; - } - - public void setShowAdvancedDialog(boolean showAdvancedDialog) { - this.showAdvancedDialog = showAdvancedDialog; - getConfig().setProperty(SHOW_ADV_DIALOG, this.showAdvancedDialog); - } - - /** - * Tells whether or not the "Referer" header should be sent in spider requests. - * - * @return {@code true} if the "Referer" header should be sent in spider requests, {@code false} - * otherwise - * @since 2.4.0 - */ - public boolean isSendRefererHeader() { - return sendRefererHeader; - } - - /** - * Sets whether or not the "Referer" header should be sent in spider requests. - * - * @param send {@code true} if the "Referer" header should be sent in spider requests, {@code - * false} otherwise - * @since 2.4.0 - */ - public void setSendRefererHeader(boolean send) { - if (send == sendRefererHeader) { - return; - } - - this.sendRefererHeader = send; - getConfig().setProperty(SPIDER_SENDER_REFERER_HEADER, this.sendRefererHeader); - } - - /** - * Returns the maximum duration in minutes that the spider should run for. Zero means no limit. - * - * @return the maximum time, in minutes, that the spider should run - */ - public int getMaxDuration() { - return maxDuration; - } - - /** - * Sets the maximum duration in minutes that the spider should run for. Zero means no limit. - * - * @param maxDuration the maximum time, in minutes, that the spider should run - */ - public void setMaxDuration(int maxDuration) { - this.maxDuration = maxDuration; - getConfig().setProperty(MAX_DURATION, maxDuration); - } - - /** - * Gets the maximum number of child nodes (per node) that can be crawled, 0 means no limit. - * - * @return the maximum number of child nodes that can be crawled. - * @since 2.6.0 - */ - public int getMaxChildren() { - return maxChildren; - } - - /** - * Sets the maximum number of child nodes (per node) that can be crawled, 0 means no limit. - * - * @param maxChildren the maximum number of child nodes that can be crawled. - * @since 2.6.0 - */ - public void setMaxChildren(int maxChildren) { - this.maxChildren = maxChildren; - getConfig().setProperty(MAX_CHILDREN, maxChildren); - } - - /** - * Sets whether or not a spider process should accept cookies while spidering. - * - *

For example, this might control whether or not the Spider uses the same session throughout - * a spidering process. - * - *

Notes: - * - *

    - *
  • This option has low priority, the Spider will respect other (global) options related to - * the HTTP state. This option is ignored if, for example, a {@link - * org.zaproxy.zap.users.User User} was set or the option Enable (Global) HTTP State is - * enabled. - *
  • The cookies are not shared between spider processes, each process has its own cookie - * jar. - *
- * - * @param acceptCookies {@code true} if the spider should accept cookies, {@code false} - * otherwise. - * @since 2.7.0 - * @see #isAcceptCookies() - */ - public void setAcceptCookies(boolean acceptCookies) { - this.acceptCookies = acceptCookies; - getConfig().setProperty(SPIDER_ACCEPT_COOKIES, acceptCookies); - } - - /** - * Tells whether or not a spider process should accept cookies while spidering. - * - *

For example, this might control whether or not the Spider uses the same session throughout - * a spidering process. - * - * @return {@code true} if the spider should accept cookies, {@code false} otherwise. - * @since 2.7.0 - * @see #setAcceptCookies(boolean) - */ - public boolean isAcceptCookies() { - return acceptCookies; - } - - /** - * Sets the maximum size, in bytes, that a response might have to be parsed. - * - *

This allows the spider to skip big responses/files. - * - * @param maxParseSizeBytes the maximum size, in bytes, that a response might have to be parsed. - * @since 2.7.0 - * @see #getMaxParseSizeBytes() - */ - public void setMaxParseSizeBytes(int maxParseSizeBytes) { - this.maxParseSizeBytes = maxParseSizeBytes; - getConfig().setProperty(SPIDER_MAX_PARSE_SIZE_BYTES, maxParseSizeBytes); - } - - /** - * Gets the maximum size, in bytes, that a response might have to be parsed. - * - * @return the maximum size, in bytes, that a response might have to be parsed. - * @since 2.7.0 - * @see #setMaxParseSizeBytes(int) - */ - public int getMaxParseSizeBytes() { - return maxParseSizeBytes; - } - - public Set getIrrelevantUrlParameters() { - return irrelevantUrlParameters; - } - - public void setIrrelevantUrlParameters(Set irrelevantUrlParameters) { - this.irrelevantUrlParameters = irrelevantUrlParameters; - getConfig().setProperty(SPIDER_IRRELEVANT_URL_PARAMETERS, irrelevantUrlParameters); - } - - public String getIrrelevantUrlParametersAsString() { - return this.getIrrelevantUrlParameters().stream().collect(Collectors.joining(", ")); - } - - public void setIrrelevantUrlParameters(String irrelevantUrlParameters) { - this.setIrrelevantUrlParameters( - (Set) - Arrays.stream(irrelevantUrlParameters.split(",")) - .map(String::trim) - .filter(str -> !str.isEmpty()) - .collect(Collectors.toCollection(LinkedHashSet::new))); - } - - public boolean isIrrelevantUrlParameter(String name) { - return getIrrelevantUrlParameters().contains(name) - || name.startsWith("utm_") - || isSessionToken(name); - } - - private static boolean isSessionToken(String paramName) { - if (extensionHttpSessions == null) { - return false; - } - return extensionHttpSessions.isDefaultSessionToken(paramName); - } - - private Set getStringSet(String key, Set defaultSet) { - try { - List parsedList = getConfig().getList(key, Collections.emptyList()); - return parsedList.isEmpty() - ? defaultSet - : parsedList.stream() - .map(Object::toString) - .collect(Collectors.toCollection(LinkedHashSet::new)); - } catch (ConversionException e) { - logConversionException(key, e); - } - return defaultSet; - } -} diff --git a/zap/src/main/java/org/zaproxy/zap/spider/SpiderTask.java b/zap/src/main/java/org/zaproxy/zap/spider/SpiderTask.java deleted file mode 100644 index 4a17e6e18ca..00000000000 --- a/zap/src/main/java/org/zaproxy/zap/spider/SpiderTask.java +++ /dev/null @@ -1,447 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2012 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.spider; - -import java.io.IOException; -import java.net.ConnectException; -import java.net.SocketException; -import java.net.SocketTimeoutException; -import java.net.UnknownHostException; -import java.nio.charset.StandardCharsets; -import java.util.List; -import javax.net.ssl.SSLException; -import net.htmlparser.jericho.Source; -import org.apache.commons.httpclient.URI; -import org.apache.commons.httpclient.URIException; -import org.apache.commons.lang3.exception.ExceptionUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.parosproxy.paros.Constant; -import org.parosproxy.paros.control.Control; -import org.parosproxy.paros.db.DatabaseException; -import org.parosproxy.paros.extension.history.ExtensionHistory; -import org.parosproxy.paros.model.HistoryReference; -import org.parosproxy.paros.network.HttpHeader; -import org.parosproxy.paros.network.HttpHeaderField; -import org.parosproxy.paros.network.HttpMalformedHeaderException; -import org.parosproxy.paros.network.HttpMessage; -import org.parosproxy.paros.network.HttpRequestHeader; -import org.parosproxy.paros.network.HttpResponseHeader; - -/** - * The SpiderTask representing a spidering task performed during the Spidering process. - * - * @deprecated (2.12.0) See the spider add-on in zap-extensions instead. - */ -@Deprecated -public class SpiderTask implements Runnable { - - /** The parent spider. */ - private Spider parent; - - /** - * The history reference to the database record where the request message has been partially - * filled in. - * - *

Might be {@code null} if failed to create or persist the message, if the task was already - * executed or if a clean up was performed. - * - * @see #cleanup() - * @see #deleteHistoryReference() - * @see #prepareHttpMessage() - */ - private HistoryReference reference; - - /** The spider resource found. */ - private org.zaproxy.zap.spider.parser.SpiderResourceFound resourceFound; - - private ExtensionHistory extHistory = null; - - /** The Constant log. */ - private static final Logger log = LogManager.getLogger(SpiderTask.class); - - /** - * Instantiates a new spider task using the target URI. The purpose of this task is to crawl the - * given uri, using the provided method and supplied request headers, find any other uris in the - * fetched resource and create other tasks. - * - *

The body of the request message is also provided in the {@literal requestBody} parameter - * and will be used when fetching the resource from the specified uri. - * - * @param parent the spider controlling the crawling process - * @param resourceFound the spider resource found - * @param uri the uri that this task should process - * @since 2.11.0 - */ - public SpiderTask( - Spider parent, - org.zaproxy.zap.spider.parser.SpiderResourceFound resourceFound, - URI uri) { - super(); - this.parent = parent; - this.resourceFound = resourceFound; - - // Log the new task - log.debug("New task submitted for uri: {}", uri); - - // Create a new HttpMessage that will be used for the request and persist it in the database - // using - // HistoryReference - try { - HttpRequestHeader requestHeader = - new HttpRequestHeader(resourceFound.getMethod(), uri, HttpHeader.HTTP11); - // Intentionally adding supplied request headers before the referer header - // to prioritize "send referer header" option - for (HttpHeaderField header : resourceFound.getHeaders()) { - requestHeader.addHeader(header.getName(), header.getValue()); - } - if (resourceFound.getMessage() != null - && parent.getSpiderParam().isSendRefererHeader()) { - requestHeader.setHeader( - HttpRequestHeader.REFERER, - resourceFound.getMessage().getRequestHeader().getURI().toString()); - } - HttpMessage msg = new HttpMessage(requestHeader); - if (!resourceFound.getBody().isEmpty()) { - msg.getRequestHeader().setContentLength(resourceFound.getBody().length()); - msg.setRequestBody(resourceFound.getBody()); - } - this.reference = - new HistoryReference( - parent.getModel().getSession(), HistoryReference.TYPE_SPIDER_TASK, msg); - } catch (HttpMalformedHeaderException e) { - log.error("Error while building HttpMessage for uri: {}", uri, e); - } catch (DatabaseException e) { - log.error("Error while persisting HttpMessage for uri: {}", uri, e); - } - } - - @Override - public void run() { - try { - if (reference == null) { - log.warn("Null URI. Skipping crawling task: {}", this); - return; - } - - log.debug( - "Spider Task Started. Processing uri at depth {} using already constructed message: {}", - resourceFound.getDepth(), - reference.getURI()); - - runImpl(); - } finally { - parent.postTaskExecution(); - log.debug("Spider Task finished."); - } - } - - private void runImpl() { - // Check if the should stop - if (parent.isStopped()) { - log.debug("Spider process is stopped. Skipping crawling task..."); - deleteHistoryReference(); - return; - } - - // Check if the crawling process is paused and do any "before execution" processing - parent.preTaskExecution(); - - // Fetch the resource - HttpMessage msg; - try { - msg = prepareHttpMessage(); - } catch (Exception e) { - log.error("Failed to prepare HTTP message: ", e); - return; - } - - try { - fetchResource(msg); - } catch (Exception e) { - setErrorResponse(msg, e); - parent.notifyListenersSpiderTaskResult( - new SpiderTaskResult(msg, getSkippedMessage("ioerror"))); - return; - } - - // Check if the should stop - if (parent.isStopped()) { - parent.notifyListenersSpiderTaskResult( - new SpiderTaskResult(msg, getSkippedMessage("stopped"))); - log.debug("Spider process is stopped. Skipping crawling task..."); - return; - } - // Check if the crawling process is paused - parent.checkPauseAndWait(); - - // Check the parse filters to see if the resource should be skipped from parsing - org.zaproxy.zap.spider.filters.ParseFilter.FilterResult filterResult = - org.zaproxy.zap.spider.filters.ParseFilter.FilterResult.NOT_FILTERED; - boolean wanted = false; - for (org.zaproxy.zap.spider.filters.ParseFilter filter : - parent.getController().getParseFilters()) { - filterResult = filter.filtered(msg); - if (filterResult.isFiltered()) { - break; - } else if (filterResult - == org.zaproxy.zap.spider.filters.ParseFilter.FilterResult.WANTED) - wanted = true; - } - if (!wanted && !filterResult.isFiltered()) { - filterResult = parent.getController().getDefaultParseFilter().filtered(msg); - } - if (filterResult.isFiltered()) { - log.debug( - "Resource [{}] fetched, but will not be parsed due to a ParseFilter rule: {}", - msg.getRequestHeader().getURI(), - filterResult.getReason()); - - parent.notifyListenersSpiderTaskResult( - new SpiderTaskResult(msg, filterResult.getReason())); - return; - } - - // Check if the should stop - if (parent.isStopped()) { - parent.notifyListenersSpiderTaskResult( - new SpiderTaskResult(msg, getSkippedMessage("stopped"))); - log.debug("Spider process is stopped. Skipping crawling task..."); - return; - } - // Check if the crawling process is paused - parent.checkPauseAndWait(); - - int maxDepth = parent.getSpiderParam().getMaxDepth(); - if (maxDepth == SpiderParam.UNLIMITED_DEPTH || resourceFound.getDepth() < maxDepth) { - parent.notifyListenersSpiderTaskResult(new SpiderTaskResult(msg)); - processResource(msg); - } else { - parent.notifyListenersSpiderTaskResult( - new SpiderTaskResult(msg, getSkippedMessage("maxdepth"))); - } - } - - private String getSkippedMessage(String key) { - return parent.getExtensionSpider() - .getMessages() - .getString("spider.task.message.skipped." + key); - } - - /** - * Prepares the HTTP message to be sent to the target server. - * - *

The HTTP message is read from the database and set up with common headers (e.g. - * User-Agent) and properties (e.g. user). - * - * @return the HTTP message - * @throws HttpMalformedHeaderException if an error occurred while parsing the HTTP message read - * from the database - * @throws DatabaseException if an error occurred while reading the HTTP message from the - * database - */ - private HttpMessage prepareHttpMessage() - throws HttpMalformedHeaderException, DatabaseException { - // Build fetch the request message from the database - HttpMessage msg; - try { - msg = reference.getHttpMessage(); - // HistoryReference is about to be deleted, so no point keeping referencing it. - msg.setHistoryRef(null); - } finally { - deleteHistoryReference(); - } - - msg.getRequestHeader().setHeader(HttpHeader.IF_MODIFIED_SINCE, null); - msg.getRequestHeader().setHeader(HttpHeader.IF_NONE_MATCH, null); - - // Check if there is a custom user agent - if (parent.getSpiderParam().getUserAgent() != null) { - msg.getRequestHeader() - .setHeader(HttpHeader.USER_AGENT, parent.getSpiderParam().getUserAgent()); - } - - // Check if there's a need to send the message from the point of view of a User - if (parent.getScanUser() != null) { - msg.setRequestingUser(parent.getScanUser()); - } - return msg; - } - - /** - * Deletes the history reference, should be called when no longer needed. - * - *

The call to this method has no effect if the history reference no longer exists (i.e. - * {@code null}). - * - * @see #reference - */ - private void deleteHistoryReference() { - if (reference == null) { - return; - } - - if (getExtensionHistory() != null) { - getExtensionHistory().delete(reference); - reference = null; - } - } - - private void setErrorResponse(HttpMessage msg, Exception cause) { - StringBuilder strBuilder = new StringBuilder(250); - if (cause instanceof SSLException) { - strBuilder.append(Constant.messages.getString("network.ssl.error.connect")); - strBuilder.append(msg.getRequestHeader().getURI().toString()).append('\n'); - strBuilder - .append(Constant.messages.getString("network.ssl.error.exception")) - .append(cause.getMessage()) - .append('\n'); - strBuilder - .append(Constant.messages.getString("network.ssl.error.exception.rootcause")) - .append(ExceptionUtils.getRootCauseMessage(cause)) - .append('\n'); - strBuilder.append( - Constant.messages.getString( - "network.ssl.error.help", - Constant.messages.getString("network.ssl.error.help.url"))); - - strBuilder.append("\n\nStack Trace:\n"); - for (String stackTraceFrame : ExceptionUtils.getRootCauseStackTrace(cause)) { - strBuilder.append(stackTraceFrame).append('\n'); - } - } else { - strBuilder - .append(cause.getClass().getName()) - .append(": ") - .append(cause.getLocalizedMessage()) - .append("\n\nStack Trace:\n"); - for (String stackTraceFrame : ExceptionUtils.getRootCauseStackTrace(cause)) { - strBuilder.append(stackTraceFrame).append('\n'); - } - } - - String message = strBuilder.toString(); - - HttpResponseHeader responseHeader; - try { - responseHeader = new HttpResponseHeader("HTTP/1.1 400 ZAP IO Error"); - responseHeader.setHeader(HttpHeader.CONTENT_TYPE, "text/plain; charset=UTF-8"); - responseHeader.setHeader( - HttpHeader.CONTENT_LENGTH, - Integer.toString(message.getBytes(StandardCharsets.UTF_8).length)); - msg.setResponseHeader(responseHeader); - msg.setResponseBody(message); - } catch (HttpMalformedHeaderException e) { - log.error("Failed to create error response:", e); - } - } - - /** - * Process a resource, searching for links (uris) to other resources. - * - * @param message the HTTP Message - */ - private void processResource(HttpMessage message) { - List parsers = - parent.getController().getParsers(); - - // Prepare the Jericho source - Source source = new Source(message.getResponseBody().toString()); - - // Get the full path of the file - String path = null; - try { - path = message.getRequestHeader().getURI().getPath(); - } catch (URIException e) { - } finally { - // Handle null paths. - if (path == null) path = ""; - } - - // Parse the resource - boolean alreadyConsumed = false; - for (org.zaproxy.zap.spider.parser.SpiderParser parser : parsers) { - if (parser.canParseResource(message, path, alreadyConsumed)) { - log.debug("Parser {} can parse resource '{}'", parser, path); - if (parser.parseResource(message, source, resourceFound.getDepth())) { - alreadyConsumed = true; - } - } else { - log.debug("Parser {} cannot parse resource '{}'", parser, path); - } - } - } - - private ExtensionHistory getExtensionHistory() { - if (this.extHistory == null) { - this.extHistory = - Control.getSingleton() - .getExtensionLoader() - .getExtension(ExtensionHistory.class); - } - return this.extHistory; - } - - /** - * Fetches a resource. - * - * @param msg the HTTP message that will be sent to the server - * @throws IOException Signals that an I/O exception has occurred. - */ - private void fetchResource(HttpMessage msg) throws IOException { - if (parent.getHttpSender() == null) { - return; - } - - try { - parent.getHttpSender().sendAndReceive(msg); - } catch (ConnectException e) { - log.debug("Failed to connect to: {}", msg.getRequestHeader().getURI(), e); - throw e; - } catch (SocketTimeoutException e) { - log.debug("Socket timeout: {}", msg.getRequestHeader().getURI(), e); - throw e; - } catch (SocketException e) { - log.debug("Socket exception: {}", msg.getRequestHeader().getURI(), e); - throw e; - } catch (UnknownHostException e) { - log.debug("Unknown host: {}", msg.getRequestHeader().getURI(), e); - throw e; - } catch (Exception e) { - log.error( - "An error occurred while fetching the resource [{}]: {}", - msg.getRequestHeader().getURI(), - e.getMessage(), - e); - throw e; - } - } - - /** - * Cleans up the resources used by the task. - * - *

Should be called if the task was not executed. - * - * @since 2.5.0 - */ - void cleanup() { - deleteHistoryReference(); - } -} diff --git a/zap/src/main/java/org/zaproxy/zap/spider/SpiderTaskResult.java b/zap/src/main/java/org/zaproxy/zap/spider/SpiderTaskResult.java deleted file mode 100644 index 7e885679511..00000000000 --- a/zap/src/main/java/org/zaproxy/zap/spider/SpiderTaskResult.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2017 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.spider; - -import org.parosproxy.paros.network.HttpMessage; - -/** - * A result from a {@link SpiderTask}. - * - *

Contains the message and processing details. - * - * @since 2.7.0 - * @deprecated (2.12.0) See the spider add-on in zap-extensions instead. - */ -@Deprecated -public class SpiderTaskResult { - - private final HttpMessage httpMessage; - private final boolean processed; - private final String reasonNotProcessed; - - /** - * Constructs a {@code SpiderTaskResult} with the given processed message. - * - * @param httpMessage the HTTP message that resulted from the spider task. - */ - public SpiderTaskResult(HttpMessage httpMessage) { - this(httpMessage, true, ""); - } - - /** - * Constructs a {@code SpiderTaskResult} with the given non-processed message and the reason of - * why it as not processed. - * - * @param httpMessage the HTTP message that resulted from the spider task. - * @param reasonNotProcessed the reason of why the HTTP message was not processed. - * @throws IllegalArgumentException if the given reason is {@code null}. - */ - public SpiderTaskResult(HttpMessage httpMessage, String reasonNotProcessed) { - this(httpMessage, false, reasonNotProcessed); - } - - /** - * Constructs a {@code SpiderTaskResult} with the given message, processed state, and the reason - * of why it as not processed. - * - * @param httpMessage the HTTP message that resulted from the spider task. - * @param processed {@code true} if the message was processed, {@code false} otherwise. - * @param reasonNotProcessed the reason of why the HTTP message was not processed. - */ - private SpiderTaskResult( - HttpMessage httpMessage, boolean processed, String reasonNotProcessed) { - if (reasonNotProcessed == null) { - throw new IllegalArgumentException("Parameter reason must not be null."); - } - this.httpMessage = httpMessage; - this.processed = processed; - this.reasonNotProcessed = reasonNotProcessed; - } - - /** - * Gets the HTTP message that resulted from a spider task. - * - * @return the HTTP message, never {@code null}. - */ - public HttpMessage getHttpMessage() { - return httpMessage; - } - - /** - * Tells whether or not the HTTP message was processed (obtained and parsed). - * - * @return {@code true} if the message was processed, {@code false} otherwise. - */ - public boolean isProcessed() { - return processed; - } - - /** - * Gets the reason of why the HTTP message was not processed. - * - * @return the reason of why the HTTP message was not processed, never {@code null}. - */ - public String getReasonNotProcessed() { - return reasonNotProcessed; - } -} diff --git a/zap/src/main/java/org/zaproxy/zap/spider/URLCanonicalizer.java b/zap/src/main/java/org/zaproxy/zap/spider/URLCanonicalizer.java deleted file mode 100644 index f4006ea1c62..00000000000 --- a/zap/src/main/java/org/zaproxy/zap/spider/URLCanonicalizer.java +++ /dev/null @@ -1,628 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * ZAP: Based on work by Yasser Ganjisaffar - * from project http://code.google.com/p/crawler4j/ - */ -package org.zaproxy.zap.spider; - -import java.net.URI; -import java.net.URL; -import java.util.HashSet; -import java.util.Set; -import java.util.SortedSet; -import java.util.TreeSet; -import java.util.function.Predicate; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import org.apache.commons.httpclient.URIException; -import org.apache.commons.httpclient.util.URIUtil; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -/** - * The URLCanonicalizer is used for the process of converting an URL into a canonical (normalized) - * form. See URL Normalization for a - * reference. - * - *

Note: some parts of the code are adapted from: stackoverflow - * - *

Added support for OData URLs - * - * @deprecated (2.12.0) See the spider add-on in zap-extensions instead. - */ -@Deprecated -public final class URLCanonicalizer { - - /** The Constant log. */ - private static final Logger log = LogManager.getLogger(URLCanonicalizer.class); - - private static final String HTTP_SCHEME = "http"; - private static final int HTTP_DEFAULT_PORT = 80; - - private static final String HTTPS_SCHEME = "https"; - private static final int HTTPS_DEFAULT_PORT = 443; - - /** - * The Constant IRRELEVANT_PARAMETERS defining the parameter names which are ignored in the URL. - */ - private static final Set IRRELEVANT_PARAMETERS = new HashSet<>(3); - - static { - IRRELEVANT_PARAMETERS.add("jsessionid"); - IRRELEVANT_PARAMETERS.add("phpsessid"); - IRRELEVANT_PARAMETERS.add("aspsessionid"); - } - - private static final Predicate DEFAULT_IRRELEVANT_PARAMETERS = - name -> IRRELEVANT_PARAMETERS.contains(name) || name.startsWith("utm_"); - - /** - * OData support Extract the ID of a resource including the surrounding quote First group is the - * resource_name Second group is the ID (quote will be taken as part of the value) - */ - private static final Pattern patternResourceIdentifierUnquoted = - Pattern.compile("/([\\w%]*)\\(([\\w']*)\\)"); - - /** OData support Detect a section containing a composite IDs */ - private static final Pattern patternResourceMultipleIdentifier = - Pattern.compile("/[\\w%]*\\((.*)\\)"); - - /** OData support Extract the detail of the multiples IDs */ - private static final Pattern patternResourceMultipleIdentifierDetail = - Pattern.compile("([\\w%]*)=([\\w']*)"); - - /** Private constructor to avoid initialization of object. */ - private URLCanonicalizer() {} - - /** - * Gets the canonical url. - * - * @param url the url - * @return the canonical url - */ - public static String getCanonicalURL(String url) { - return getCanonicalURL(url, null); - } - - /** - * Gets the canonical url, starting from a relative or absolute url found in a given context - * (baseURL). - * - * @param url the url string defining the reference - * @param baseURL the context in which this url was found - * @return the canonical url - */ - public static String getCanonicalURL(String url, String baseURL) { - return getCanonicalURL(url, baseURL, DEFAULT_IRRELEVANT_PARAMETERS); - } - - /** - * Gets the canonical url, starting from a relative or absolute url found in a given context - * (baseURL). - * - * @param url the url string defining the reference - * @param baseURL the context in which this url was found - * @param irrelevantParameter a predicate to ignore parameters. - * @return the canonical url - */ - public static String getCanonicalURL( - String url, String baseURL, Predicate irrelevantParameter) { - if ("javascript:".equals(url)) { - return null; - } - - try { - /* Build the absolute URL, from the url and the baseURL */ - String resolvedURL = URLResolver.resolveUrl(baseURL == null ? "" : baseURL, url); - log.debug("Resolved URL: {}", resolvedURL); - URI canonicalURI; - try { - canonicalURI = new URI(resolvedURL); - } catch (Exception e) { - canonicalURI = new URI(URIUtil.encodeQuery(resolvedURL)); - } - - /* Some checking. */ - if (canonicalURI.getScheme() == null) { - log.warn( - "Protocol could not be reliably evaluated from uri: {} and base url: {}", - canonicalURI, - baseURL); - return null; - } - - if (canonicalURI.getRawAuthority() == null) { - log.debug( - "Ignoring URI with no authority (host[\":\"port]): {} (on base {})", - canonicalURI, - baseURL); - return null; - } - - if (canonicalURI.getHost() == null) { - log.warn( - "Host could not be reliably evaluated from: {} (on base {})", - canonicalURI, - baseURL); - return null; - } - - /* - * Normalize: no empty segments (i.e., "//"), no segments equal to ".", and no segments equal to - * ".." that are preceded by a segment not equal to "..". - */ - String path = canonicalURI.normalize().getRawPath(); - - /* Convert '//' -> '/' */ - int idx = path.indexOf("//"); - while (idx >= 0) { - path = path.replace("//", "/"); - idx = path.indexOf("//"); - } - - /* Drop starting '/../' */ - while (path.startsWith("/../")) { - path = path.substring(3); - } - - /* Trim */ - path = path.trim(); - - /* Process parameters and sort them. */ - final SortedSet params = - createSortedParameters(canonicalURI.getRawQuery()); - final String queryString; - String canonicalParams = canonicalize(params, irrelevantParameter); - queryString = (canonicalParams.isEmpty() ? "" : "?" + canonicalParams); - - /* Add starting slash if needed */ - if (path.length() == 0) { - path = "/" + path; - } - - /* Drop default port: example.com:80 -> example.com */ - int port = canonicalURI.getPort(); - if (isDefaultPort(canonicalURI.getScheme(), port)) { - port = -1; - } - - /* Lowercasing protocol and host */ - String protocol = canonicalURI.getScheme().toLowerCase(); - String host = canonicalURI.getHost().toLowerCase(); - String pathAndQueryString = normalizePath(path) + queryString; - - URL result = new URL(protocol, host, port, pathAndQueryString); - return result.toExternalForm(); - - } catch (Exception ex) { - log.warn( - "Error while Processing URL [{}" - + "] in the spidering process (on base {}): {}", - url, - baseURL, - ex.getMessage()); - return null; - } - } - - /** - * Tells whether or not the given port is the default for the given scheme. - * - *

Note: Only HTTP and HTTPS schemes are taken into account. - * - * @param scheme the scheme - * @param port the port - * @return {@code true} if given the port is the default port for the given scheme, {@code - * false} otherwise. - */ - private static boolean isDefaultPort(String scheme, int port) { - return HTTP_SCHEME.equalsIgnoreCase(scheme) && port == HTTP_DEFAULT_PORT - || HTTPS_SCHEME.equalsIgnoreCase(scheme) && port == HTTPS_DEFAULT_PORT; - } - - /** - * Builds a String representation of the URI with cleaned parameters, that can be used when - * checking if an URI was already visited. The URI provided as a parameter should be already - * cleaned and canonicalized, so it should be build with a result from {@link - * #getCanonicalURL(String)}. - * - *

When building the URI representation, the same format should be used for all the cases, as - * it may affect the number of times the pages are visited and reported if the option - * HandleParametersOption is changed while the spider is running. - * - * @param uri the uri - * @param handleParameters the handle parameters option - * @param handleODataParametersVisited Should we handle specific OData parameters - * @return the string representation of the URI - * @throws URIException the URI exception - */ - public static String buildCleanedParametersURIRepresentation( - org.apache.commons.httpclient.URI uri, - SpiderParam.HandleParametersOption handleParameters, - boolean handleODataParametersVisited) - throws URIException { - return buildCleanedParametersURIRepresentation( - uri, handleParameters, handleODataParametersVisited, DEFAULT_IRRELEVANT_PARAMETERS); - } - - /** - * Builds a String representation of the URI with cleaned parameters, that can be used when - * checking if an URI was already visited. The URI provided as a parameter should be already - * cleaned and canonicalized, so it should be build with a result from {@link - * #getCanonicalURL(String)}. - * - *

When building the URI representation, the same format should be used for all the cases, as - * it may affect the number of times the pages are visited and reported if the option - * HandleParametersOption is changed while the spider is running. - * - * @param uri the uri - * @param handleParameters the handle parameters option - * @param handleODataParametersVisited Should we handle specific OData parameters - * @param irrelevantParameter a predicate to ignore parameters. - * @return the string representation of the URI - * @throws URIException the URI exception - */ - static String buildCleanedParametersURIRepresentation( - org.apache.commons.httpclient.URI uri, - SpiderParam.HandleParametersOption handleParameters, - boolean handleODataParametersVisited, - Predicate irrelevantParameter) - throws URIException { - // If the option is set to use all the information, just use the default string - // representation - if (handleParameters.equals( - org.zaproxy.zap.spider.SpiderParam.HandleParametersOption.USE_ALL)) { - return uri.toString(); - } - - // If the option is set to ignore parameters completely, ignore the query completely - if (handleParameters.equals( - org.zaproxy.zap.spider.SpiderParam.HandleParametersOption.IGNORE_COMPLETELY)) { - return createBaseUriWithCleanedPath( - uri, handleParameters, handleODataParametersVisited); - } - - // If the option is set to ignore the value, we get the parameters and we only add their - // name to the - // query - if (handleParameters.equals( - org.zaproxy.zap.spider.SpiderParam.HandleParametersOption.IGNORE_VALUE)) { - StringBuilder retVal = - new StringBuilder( - createBaseUriWithCleanedPath( - uri, handleParameters, handleODataParametersVisited)); - - String cleanedQuery = getCleanedQuery(uri.getEscapedQuery(), irrelevantParameter); - - // Add the parameters' names to the uri representation. - if (cleanedQuery.length() > 0) { - retVal.append('?').append(cleanedQuery); - } - - return retVal.toString(); - } - - // Should not be reached - return uri.toString(); - } - - private static String createBaseUriWithCleanedPath( - org.apache.commons.httpclient.URI uri, - org.zaproxy.zap.spider.SpiderParam.HandleParametersOption handleParameters, - boolean handleODataParametersVisited) - throws URIException { - StringBuilder uriBuilder = new StringBuilder(createBaseUri(uri)); - - uriBuilder.append( - getCleanedPath( - uri.getEscapedPath(), handleParameters, handleODataParametersVisited)); - - return uriBuilder.toString(); - } - - private static String createBaseUri(org.apache.commons.httpclient.URI uri) throws URIException { - StringBuilder baseUriBuilder = new StringBuilder(); - baseUriBuilder.append(uri.getScheme()).append("://").append(uri.getHost()); - if (uri.getPort() != -1) { - baseUriBuilder.append(':').append(uri.getPort()); - } - return baseUriBuilder.toString(); - } - - private static String getCleanedPath( - String escapedPath, - org.zaproxy.zap.spider.SpiderParam.HandleParametersOption handleParameters, - boolean handleODataParametersVisited) { - if (escapedPath == null) { - return ""; - } - - String cleanedPath; - if (handleODataParametersVisited) { - cleanedPath = cleanODataPath(escapedPath, handleParameters); - } else { - cleanedPath = escapedPath; - } - - return cleanedPath; - } - - private static String getCleanedQuery( - String escapedQuery, Predicate irrelevantParameter) { - // Get the parameters' names - SortedSet params = createSortedParameters(escapedQuery); - Set parameterNames = new HashSet<>(); - StringBuilder cleanedQueryBuilder = new StringBuilder(); - if (params != null && !params.isEmpty()) { - for (QueryParameter parameter : params) { - String name = parameter.getName(); - if (parameterNames.contains(name)) { - continue; - } - parameterNames.add(name); - if (irrelevantParameter.test(name)) { - continue; - } - if (cleanedQueryBuilder.length() > 0) { - cleanedQueryBuilder.append('&'); - } - cleanedQueryBuilder.append(name); - } - } - - return cleanedQueryBuilder.toString(); - } - - /** - * Clean the path in the case of an OData Uri containing a resource identifier (simple or - * multiple) - * - * @param path The path to clean - * @param handleParameters tThe cleaning mode - * @return A cleaned path - */ - private static String cleanODataPath( - String path, - org.zaproxy.zap.spider.SpiderParam.HandleParametersOption handleParameters) { - String cleanedPath = path; - - if (org.zaproxy.zap.spider.SpiderParam.HandleParametersOption.USE_ALL.equals( - handleParameters)) { - cleanedPath = path; - } else { - - // check for single ID (unnamed) - Matcher matcher = patternResourceIdentifierUnquoted.matcher(path); - if (matcher.find()) { - String resourceName = matcher.group(1); - String resourceID = matcher.group(2); - - String subString = resourceName + "(" + resourceID + ")"; - int begin = path.indexOf(subString); - int end = begin + subString.length(); - - String beforeSubstring = path.substring(0, begin); - String afterSubstring = path.substring(end); - - if (org.zaproxy.zap.spider.SpiderParam.HandleParametersOption.IGNORE_COMPLETELY - .equals(handleParameters) - || org.zaproxy.zap.spider.SpiderParam.HandleParametersOption.IGNORE_VALUE - .equals(handleParameters)) { - - StringBuilder sb = new StringBuilder(beforeSubstring); - sb.append(resourceName).append("()").append(afterSubstring); - cleanedPath = sb.toString(); - } - - } else { - - matcher = patternResourceMultipleIdentifier.matcher(path); - if (matcher.find()) { - // We've found a composite identifier. i.e: /Resource(field1=a,field2=3) - - String multipleIdentifierSection = matcher.group(1); - - int begin = path.indexOf(multipleIdentifierSection); - int end = begin + multipleIdentifierSection.length(); - - String beforeSubstring = path.substring(0, begin); - String afterSubstring = path.substring(end); - - if (org.zaproxy.zap.spider.SpiderParam.HandleParametersOption.IGNORE_COMPLETELY - .equals(handleParameters)) { - cleanedPath = beforeSubstring + afterSubstring; - } else { - StringBuilder sb = new StringBuilder(beforeSubstring); - - matcher = - patternResourceMultipleIdentifierDetail.matcher( - multipleIdentifierSection); - int i = 1; - while (matcher.find()) { - - if (i > 1) { - sb.append(','); - } - String paramName = matcher.group(1); - sb.append(paramName); - i++; - } - - sb.append(afterSubstring); - cleanedPath = sb.toString(); - } - } - } - } - - return cleanedPath; - } - - /** - * Creates a sorted set with all the parameters from the given {@code query}, ordered - * lexicographically by name and value. - * - * @param queryString the query string - * @return a sorted set with all parameters, or {@code null} if the query string is {@code null} - * or empty. - */ - private static SortedSet createSortedParameters(final String queryString) { - if (queryString == null || queryString.isEmpty()) { - return null; - } - - final String[] pairs = queryString.split("&"); - final SortedSet params = new TreeSet<>(); - - for (final String pair : pairs) { - if (pair.length() == 0) { - continue; - } - - String[] tokens = pair.split("=", 2); - switch (tokens.length) { - case 1: - if (pair.charAt(0) == '=') { - params.add(new QueryParameter("", tokens[0])); - } else { - params.add(new QueryParameter(tokens[0], "")); - } - break; - case 2: - params.add(new QueryParameter(tokens[0], tokens[1])); - break; - } - } - return params; - } - - /** - * Canonicalize the query string. - * - * @param sortedParameters Parameter name-value pairs in lexicographical order. - * @param irrelevantParameter url parameters that are skipped - * @return Canonical form of query string. - */ - private static String canonicalize( - final SortedSet sortedParameters, - Predicate irrelevantParameter) { - if (sortedParameters == null || sortedParameters.isEmpty()) { - return ""; - } - - final StringBuilder sb = new StringBuilder(100); - for (QueryParameter parameter : sortedParameters) { - final String name = parameter.getName().toLowerCase(); - if (irrelevantParameter.test(name)) { - continue; - } - if (sb.length() > 0) { - sb.append('&'); - } - sb.append(parameter.getName()); - if (!parameter.getValue().isEmpty()) { - sb.append('='); - sb.append(parameter.getValue()); - } - } - return sb.toString(); - } - - /** - * Normalize path. - * - * @param path the path - * @return the string - */ - private static String normalizePath(final String path) { - return path.replace("%7E", "~").replace(" ", "%20"); - } - - /** - * A query parameter, with non-{@code null} name and value. - * - *

The query parameters are ordered by name and value. - */ - private static class QueryParameter implements Comparable { - - private final String name; - private final String value; - - public QueryParameter(String name, String value) { - if (name == null) { - throw new IllegalArgumentException("Parameter name must not be null."); - } - if (value == null) { - throw new IllegalArgumentException("Parameter value must not be null."); - } - this.name = name; - this.value = value; - } - - public String getName() { - return name; - } - - public String getValue() { - return value; - } - - @Override - public int compareTo(QueryParameter other) { - if (other == null) { - return 1; - } - int result = name.compareTo(other.name); - if (result != 0) { - return result; - } - return value.compareTo(other.value); - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + name.hashCode(); - result = prime * result + value.hashCode(); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null) { - return false; - } - if (getClass() != obj.getClass()) { - return false; - } - QueryParameter other = (QueryParameter) obj; - if (!name.equals(other.name)) { - return false; - } - if (!value.equals(other.value)) { - return false; - } - return true; - } - } -} diff --git a/zap/src/main/java/org/zaproxy/zap/spider/URLResolver.java b/zap/src/main/java/org/zaproxy/zap/spider/URLResolver.java deleted file mode 100644 index 3d5c51f33f5..00000000000 --- a/zap/src/main/java/org/zaproxy/zap/spider/URLResolver.java +++ /dev/null @@ -1,489 +0,0 @@ -/* - * This class is adopted from Htmlunit with the following copyright: - * - * Copyright (c) 2002-2012 Gargoyle Software Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * ZAP: Based on work by Gargoyle Software and adopted from Htmlunit (as stated above). - */ -package org.zaproxy.zap.spider; - -/** - * The UrlResolve handles the process of resolving a relative URL against a base URL (context). - * - * @deprecated (2.12.0) See the spider add-on in zap-extensions instead. - */ -@Deprecated -public final class URLResolver { - - /** Private constructor to avoid initialization of object. */ - private URLResolver() {} - - /** - * Resolves a given relative URL against a base URL. See RFC1808 Section 4 for more details. - * - * @param baseUrl The base URL in which to resolve the specification. - * @param relativeUrl The relative URL to resolve against the base URL. - * @return the resolved specification. - */ - public static String resolveUrl(final String baseUrl, final String relativeUrl) { - // Parameter check - if (baseUrl == null) { - throw new IllegalArgumentException("Base URL must not be null"); - } - if (relativeUrl == null) { - throw new IllegalArgumentException("Relative URL must not be null"); - } - final Url url = resolveUrl(parseUrl(baseUrl.trim()), relativeUrl.trim()); - - return url.toString(); - } - - /** - * Returns the index within the specified string of the first occurrence of the specified search - * character. - * - * @param s the string to search - * @param searchChar the character to search for - * @param beginIndex the index at which to start the search - * @param endIndex the index at which to stop the search - * @return the index of the first occurrence of the character in the string or -1 - */ - private static int indexOf( - final String s, final char searchChar, final int beginIndex, final int endIndex) { - for (int i = beginIndex; i < endIndex; i++) { - if (s.charAt(i) == searchChar) { - return i; - } - } - return -1; - } - - /** - * Parses a given specification using the algorithm depicted in RFC1808 - Section 2.4: - * - *

- * - *

Section 2.4: Parsing a URL

- * - * An accepted method for parsing URLs is useful to clarify the generic-RL syntax of Section 2.2 - * and to describe the algorithm for resolving relative URLs presented in Section 4. This - * section describes the parsing rules for breaking down a URL (relative or absolute) into the - * component parts described in Section 2.1. The rules assume that the URL has already been - * separated from any surrounding text and copied to a "parse string". The rules are listed in - * the order in which they would be applied by the parser. - * - *
- * - * @param spec The specification to parse. - * @return the parsed specification. - */ - private static Url parseUrl(final String spec) { - final Url url = new Url(); - int startIndex = 0; - int endIndex = spec.length(); - - // Section 2.4.1: Parsing the Fragment Identifier - // - // If the parse string contains a crosshatch "#" character, then the - // substring after the first (left-most) crosshatch "#" and up to the - // end of the parse string is the identifier. If the - // crosshatch is the last character, or no crosshatch is present, then - // the fragment identifier is empty. The matched substring, including - // the crosshatch character, is removed from the parse string before - // continuing. - // - // Note that the fragment identifier is not considered part of the URL. - // However, since it is often attached to the URL, parsers must be able - // to recognize and set aside fragment identifiers as part of the - // process. - final int crosshatchIndex = indexOf(spec, '#', startIndex, endIndex); - - if (crosshatchIndex >= 0) { - url.fragment_ = spec.substring(crosshatchIndex + 1, endIndex); - endIndex = crosshatchIndex; - } - // Section 2.4.2: Parsing the Scheme - // - // If the parse string contains a colon ":" after the first character - // and before any characters not allowed as part of a scheme name (i.e., - // any not an alphanumeric, plus "+", period ".", or hyphen "-"), the - // of the URL is the substring of characters up to but not - // including the first colon. These characters and the colon are then - // removed from the parse string before continuing. - final int colonIndex = indexOf(spec, ':', startIndex, endIndex); - - if (colonIndex > 0) { - final String scheme = spec.substring(startIndex, colonIndex); - if (isValidScheme(scheme)) { - url.scheme_ = scheme; - startIndex = colonIndex + 1; - } - } - - // Section 2.4.3: Parsing the Network Location/Login - // - // If the parse string begins with a double-slash "//", then the - // substring of characters after the double-slash and up to, but not - // including, the next slash "/" character is the network location/login - // () of the URL. If no trailing slash "/" is present, the - // entire remaining parse string is assigned to . The double- - // slash and are removed from the parse string before - // continuing. - // - // Note: We also accept a question mark "?" or a semicolon ";" character as - // delimiters for the network location/login () of the URL. - final int locationStartIndex; - int locationEndIndex; - - if (spec.startsWith("//", startIndex)) { - locationStartIndex = startIndex + 2; - locationEndIndex = indexOf(spec, '/', locationStartIndex, endIndex); - if (locationEndIndex >= 0) { - startIndex = locationEndIndex; - } - } else { - locationStartIndex = -1; - locationEndIndex = -1; - } - // Section 2.4.4: Parsing the Query Information - // - // If the parse string contains a question mark "?" character, then the - // substring after the first (left-most) question mark "?" and up to the - // end of the parse string is the information. If the question - // mark is the last character, or no question mark is present, then the - // query information is empty. The matched substring, including the - // question mark character, is removed from the parse string before - // continuing. - final int questionMarkIndex = indexOf(spec, '?', startIndex, endIndex); - - if (questionMarkIndex >= 0) { - if ((locationStartIndex >= 0) && (locationEndIndex < 0)) { - // The substring of characters after the double-slash and up to, but not - // including, the question mark "?" character is the network location/login - // () of the URL. - locationEndIndex = questionMarkIndex; - startIndex = questionMarkIndex; - } - url.query_ = spec.substring(questionMarkIndex + 1, endIndex); - endIndex = questionMarkIndex; - } - // Section 2.4.5: Parsing the Parameters - // - // If the parse string contains a semicolon ";" character, then the - // substring after the first (left-most) semicolon ";" and up to the end - // of the parse string is the parameters (). If the semicolon - // is the last character, or no semicolon is present, then is - // empty. The matched substring, including the semicolon character, is - // removed from the parse string before continuing. - final int semicolonIndex = indexOf(spec, ';', startIndex, endIndex); - - if (semicolonIndex >= 0) { - if ((locationStartIndex >= 0) && (locationEndIndex < 0)) { - // The substring of characters after the double-slash and up to, but not - // including, the semicolon ";" character is the network location/login - // () of the URL. - locationEndIndex = semicolonIndex; - startIndex = semicolonIndex; - } - url.parameters_ = spec.substring(semicolonIndex + 1, endIndex); - endIndex = semicolonIndex; - } - - // Section 2.4.6: Parsing the Path - // - // After the above steps, all that is left of the parse string is the URL and the - // slash "/" that may precede it. Even though the initial slash is not part of the URL path, - // the parser must remember whether or not it was present so that later processes can - // differentiate between relative and absolute paths. Often this is done by simply storing - // the preceding slash along with the path. - if ((locationStartIndex >= 0) && (locationEndIndex < 0)) { - // The entire remaining parse string is assigned to the network location/login - // () of the URL. - locationEndIndex = endIndex; - } else if (startIndex < endIndex) { - url.path_ = spec.substring(startIndex, endIndex); - } - // Set the network location/login () of the URL. - if ((locationStartIndex >= 0) && (locationEndIndex >= 0)) { - url.location_ = spec.substring(locationStartIndex, locationEndIndex); - } - - return url; - } - - /** - * Checks if it is a valid scheme. - * - * @param scheme the scheme - * @return true, if is a valid scheme name - */ - private static boolean isValidScheme(final String scheme) { - final int length = scheme.length(); - if (length <= 0) { - return false; - } - char c = scheme.charAt(0); - if (!Character.isLetter(c)) { - return false; - } - for (int i = 1; i < length; i++) { - c = scheme.charAt(i); - if (!Character.isLetterOrDigit(c) && c != '.' && c != '+' && c != '-') { - return false; - } - } - return true; - } - - /** - * Resolves a given relative URL against a base URL using the algorithm depicted in RFC1808: - * - *

Section 4: Resolving Relative URLs - * - *

This section describes an example algorithm for resolving URLs within a context in which - * the URLs may be relative, such that the result is always a URL in absolute form. Although - * this algorithm cannot guarantee that the resulting URL will equal that intended by the - * original author, it does guarantee that any valid URL (relative or absolute) can be - * consistently transformed to an absolute form given a valid base URL. - * - * @param baseUrl The base URL in which to resolve the specification. - * @param relativeUrl The relative URL to resolve against the base URL. - * @return the resolved specification. - */ - private static Url resolveUrl(final Url baseUrl, final String relativeUrl) { - final Url url = parseUrl(relativeUrl); - // Step 1: The base URL is established according to the rules of - // Section 3. If the base URL is the empty string (unknown), - // the embedded URL is interpreted as an absolute URL and - // we are done. - if (baseUrl == null) { - return url; - } - // Step 2: Both the base and embedded URLs are parsed into their component parts as - // described in Section 2.4. - // a) If the embedded URL is entirely empty, it inherits the - // entire base URL (i.e., is set equal to the base URL) and we are done. - if (relativeUrl.length() == 0) { - return new Url(baseUrl); - } - - // b) If the embedded URL starts with a scheme name, it is interpreted as an absolute URL - // and we are done. - if (url.scheme_ != null) { - return url; - } - - // c) Otherwise, the embedded URL inherits the scheme of the base URL. - url.scheme_ = baseUrl.scheme_; - - // Step 3: If the embedded URL's is non-empty, we skip to Step 7. Otherwise, the - // embedded URL inherits the (if any) of the base URL. - if (url.location_ != null) { - return url; - } - url.location_ = baseUrl.location_; - - // Step 4: If the embedded URL path is preceded by a slash "/", the - // path is not relative and we skip to Step 7. - if ((url.path_ != null) && ((url.path_.length() > 0) && ('/' == url.path_.charAt(0)))) { - url.path_ = removeLeadingSlashPoints(url.path_); - return url; - } - - // Step 5: If the embedded URL path is empty (and not preceded by a - // slash), then the embedded URL inherits the base URL path, - // and - if (url.path_ == null) { - url.path_ = baseUrl.path_; - // a) if the embedded URL's is non-empty, we skip to - // step 7; otherwise, it inherits the of the base - // URL (if any) and - if (url.parameters_ != null) { - return url; - } - url.parameters_ = baseUrl.parameters_; - - // b) if the embedded URL's is non-empty, we skip to - // step 7; otherwise, it inherits the of the base - // URL (if any) and we skip to step 7. - if (url.query_ != null) { - return url; - } - url.query_ = baseUrl.query_; - return url; - } - // Step 6: The last segment of the base URL's path (anything - // following the rightmost slash "/", or the entire path if no - // slash is present) is removed and the embedded URL's path is - // appended in its place. - final String basePath = baseUrl.path_; - String path = ""; - - if (basePath != null) { - final int lastSlashIndex = basePath.lastIndexOf('/'); - - if (lastSlashIndex >= 0) { - path = basePath.substring(0, lastSlashIndex + 1); - } - } else { - path = "/"; - } - path = path.concat(url.path_); - - // The following operations are then applied, in order, to the new path: - // a) All occurrences of "./", where "." is a complete path segment, are removed. - int pathSegmentIndex; - - while ((pathSegmentIndex = path.indexOf("/./")) >= 0) { - path = - path.substring(0, pathSegmentIndex + 1) - .concat(path.substring(pathSegmentIndex + 3)); - } - - // b) If the path ends with "." as a complete path segment, that "." is removed. - if (path.endsWith("/.")) { - path = path.substring(0, path.length() - 1); - } - - // c) All occurrences of "/../", where is a complete path segment not - // equal to "..", are removed. Removal of these path segments is performed iteratively, - // removing the leftmost matching pattern on each iteration, until no matching pattern - // remains. - while ((pathSegmentIndex = path.indexOf("/../")) > 0) { - final String pathSegment = path.substring(0, pathSegmentIndex); - final int slashIndex = pathSegment.lastIndexOf('/'); - - if (slashIndex < 0) { - continue; - } - if (!"..".equals(pathSegment.substring(slashIndex))) { - path = - path.substring(0, slashIndex + 1) - .concat(path.substring(pathSegmentIndex + 4)); - } - } - // d) If the path ends with "/..", where is a complete path segment not - // equal to "..", that "/.." is removed. - if (path.endsWith("/..")) { - final String pathSegment = path.substring(0, path.length() - 3); - final int slashIndex = pathSegment.lastIndexOf('/'); - - if (slashIndex >= 0) { - path = path.substring(0, slashIndex + 1); - } - } - path = removeLeadingSlashPoints(path); - url.path_ = path; - - // Step 7: The resulting URL components, including any inherited from - // the base URL, are recombined to give the absolute form of - // the embedded URL. - return url; - } - - /** - * "/.." at the beginning should be removed as browsers do (not in RFC). - * - * @param path the path - * @return the cleaned string defining the path. - */ - private static String removeLeadingSlashPoints(String path) { - while (path.startsWith("/..")) { - path = path.substring(3); - } - - return path; - } - - /** - * Class Url represents a Uniform Resource Locator. - * - * @author Martin Tamme - */ - private static class Url { - - /** The scheme_. */ - private String scheme_; - - /** The location_. */ - private String location_; - - /** The path_. */ - private String path_; - - /** The parameters_. */ - private String parameters_; - - /** The query_. */ - private String query_; - - /** The fragment_. */ - private String fragment_; - - /** Creates an Url object. */ - public Url() {} - - /** - * Creates an Url object from the specified Url object. - * - * @param url an Url object. - */ - public Url(final Url url) { - scheme_ = url.scheme_; - location_ = url.location_; - path_ = url.path_; - parameters_ = url.parameters_; - query_ = url.query_; - fragment_ = url.fragment_; - } - - /** - * Returns a string representation of the Url object. - * - * @return a string representation of the Url object. - */ - @Override - public String toString() { - final StringBuilder sb = new StringBuilder(); - - if (scheme_ != null) { - sb.append(scheme_); - sb.append(':'); - } - if (location_ != null) { - sb.append("//"); - sb.append(location_); - } - if (path_ != null) { - sb.append(path_); - } - if (parameters_ != null) { - sb.append(';'); - sb.append(parameters_); - } - if (query_ != null) { - sb.append('?'); - sb.append(query_); - } - if (fragment_ != null) { - sb.append('#'); - sb.append(fragment_); - } - return sb.toString(); - } - } -} diff --git a/zap/src/main/java/org/zaproxy/zap/spider/filters/DefaultFetchFilter.java b/zap/src/main/java/org/zaproxy/zap/spider/filters/DefaultFetchFilter.java deleted file mode 100644 index f360468bbfe..00000000000 --- a/zap/src/main/java/org/zaproxy/zap/spider/filters/DefaultFetchFilter.java +++ /dev/null @@ -1,191 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2012 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.spider.filters; - -import java.util.Collections; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Set; -import org.apache.commons.httpclient.URI; -import org.apache.commons.httpclient.URIException; -import org.zaproxy.zap.model.Context; - -/** - * The DefaultFetchFilter is an implementation of a FetchFilter that is default for spidering - * process. Its filter rules are the following: - * - *

    - *
  • the resource protocol/scheme must be 'HTTP' or 'HTTPs'. - *
  • the resource must be found in the scope (domain) of the spidering process. - *
  • the resource must be not be excluded by user request - exclude list. - *
- * - * @deprecated (2.12.0) See the spider add-on in zap-extensions instead. - */ -@Deprecated -public class DefaultFetchFilter extends FetchFilter { - - /** The scope. */ - private Set scopes = new LinkedHashSet<>(); - - private List domainsAlwaysInScope = - Collections.emptyList(); - - /** The exclude list. */ - private List excludeList = null; - - private Context scanContext; - - @Override - public FetchStatus checkFilter(URI uri) { - - getLogger().debug("Checking: {}", uri); - // Protocol check - String scheme = uri.getScheme(); - if (scheme == null - || (!scheme.equalsIgnoreCase("http") && !scheme.equalsIgnoreCase("https"))) { - return FetchStatus.ILLEGAL_PROTOCOL; - } - - try { - - // Context check - if (this.scanContext != null) { - if (!this.scanContext.isInContext(uri.toString())) { - return FetchStatus.OUT_OF_CONTEXT; - } - } else { - // Scope check - String host = uri.getHost(); - if (!isDomainInScope(host) && !isDomainAlwaysInScope(host)) { - return FetchStatus.OUT_OF_SCOPE; - } - } - - // Check if any of the exclusion regexes match. - if (isExcluded(uri.toString())) { - return FetchStatus.USER_RULES; - } - - } catch (URIException e) { - getLogger().warn("Error while fetching host for uri: {}", uri, e); - return FetchStatus.OUT_OF_SCOPE; - } - - return FetchStatus.VALID; - } - - /** - * Tells whether or not the given URI is excluded. - * - * @param uri the URI to check - * @return {@code true} if the URI is excluded, {@code false} otherwise. - */ - private boolean isExcluded(String uri) { - if (excludeList == null || excludeList.isEmpty()) { - return false; - } - - for (String ex : excludeList) { - if (uri.matches(ex)) { - return true; - } - } - return false; - } - - /** - * Tells whether or not the given domain is one of the domains in scope. - * - * @param domain the domain to check - * @return {@code true} if it's a domain in scope, {@code false} otherwise. - * @see #scopes - * @see #isDomainAlwaysInScope(String) - */ - private boolean isDomainInScope(String domain) { - for (String scope : scopes) { - if (domain.matches(scope)) { - return true; - } - } - return false; - } - - /** - * Tells whether or not the given domain is one of the domains always in scope. - * - * @param domain the domain to check - * @return {@code true} if it's a domain always in scope, {@code false} otherwise. - * @see #domainsAlwaysInScope - * @see #isDomainInScope(String) - */ - private boolean isDomainAlwaysInScope(String domain) { - for (org.zaproxy.zap.spider.DomainAlwaysInScopeMatcher domainInScope : - domainsAlwaysInScope) { - if (domainInScope.matches(domain)) { - return true; - } - } - return false; - } - - /** - * Adds a new domain to the scope list of the spider process. - * - * @param scope the scope - */ - public void addScopeRegex(String scope) { - this.scopes.add(scope); - } - - /** - * Sets the domains that will be considered as always in scope. - * - * @param domainsAlwaysInScope the list containing all domains that are always in scope. - * @since 2.3.0 - */ - public void setDomainsAlwaysInScope( - List domainsAlwaysInScope) { - if (domainsAlwaysInScope == null || domainsAlwaysInScope.isEmpty()) { - this.domainsAlwaysInScope = Collections.emptyList(); - } else { - this.domainsAlwaysInScope = domainsAlwaysInScope; - } - } - - /** - * Sets the regexes which are used for checking if an uri should be skipped. - * - * @param excl the new exclude regexes - */ - public void setExcludeRegexes(List excl) { - excludeList = excl; - } - - /** - * Sets the scan context. If set, only uris that are part of the context are considered valid - * for fetching. - * - * @param scanContext the new scan context - */ - public void setScanContext(Context scanContext) { - this.scanContext = scanContext; - } -} diff --git a/zap/src/main/java/org/zaproxy/zap/spider/filters/DefaultParseFilter.java b/zap/src/main/java/org/zaproxy/zap/spider/filters/DefaultParseFilter.java deleted file mode 100644 index 84d98531e1b..00000000000 --- a/zap/src/main/java/org/zaproxy/zap/spider/filters/DefaultParseFilter.java +++ /dev/null @@ -1,172 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2012 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.spider.filters; - -import java.util.Collections; -import java.util.Enumeration; -import java.util.ResourceBundle; -import java.util.regex.Pattern; -import org.parosproxy.paros.network.HttpMessage; -import org.parosproxy.paros.network.HttpStatusCode; - -/** - * The DefaultParseFilter is an implementation of a {@link ParseFilter} that is default for - * spidering process. Its filter rules are the following: - * - *
    - *
  • the resource body should be under a {@code SpiderParam#getMaxParseSizeBytes() number of - * bytes}, otherwise it's considered a binary resource. - *
  • the resource must be of parsable type (text, html, xml, javascript). Actually, the content - * type should be text/... - *
- * - * @deprecated (2.12.0) See the spider add-on in zap-extensions instead. - */ -@Deprecated -public class DefaultParseFilter extends ParseFilter { - - /** - * The Constant MAX_RESPONSE_BODY_SIZE defining the size of response body that is considered too - * big for a parsable file. - * - * @deprecated (2.7.0) No longer in use, replaced by {@code SpiderParam#getMaxParseSizeBytes()}. - */ - @Deprecated public static final int MAX_RESPONSE_BODY_SIZE = 512000; - - /** a pattern to match the SQLite based ".svn/wc.db" file name. */ - private static final Pattern SVN_SQLITE_FILENAME_PATTERN = Pattern.compile(".*/\\.svn/wc.db$"); - - /** a pattern to match the XML based ".svn/entries" file name. */ - private static final Pattern SVN_XML_FILENAME_PATTERN = Pattern.compile(".*/\\.svn/entries$"); - - /** a pattern to match the Git index file. */ - private static final Pattern GIT_FILENAME_PATTERN = Pattern.compile(".*/\\.git/index$"); - - /** a pattern to match the robots.txt file. */ - private static final Pattern ROBOTS_FILENAME_PATTERN = - Pattern.compile(".*/robots.txt$", Pattern.CASE_INSENSITIVE); - - /** a pattern to match the sitemap.xml file. */ - private static final Pattern SITEMAP_FILENAME_PATTERN = - Pattern.compile(".*/sitemap.xml$", Pattern.CASE_INSENSITIVE); - - /** The configurations of the spider, never {@code null}. */ - private final org.zaproxy.zap.spider.SpiderParam params; - - private final FilterResult filterResultEmpty; - private final FilterResult filterResultMaxSize; - private final FilterResult filterResultNotText; - - /** - * Constructs a {@code DefaultParseFilter} with default configurations. - * - * @deprecated (2.7.0) Replaced by {@code #DefaultParseFilter(SpiderParam, ResourceBundle)}. - */ - @Deprecated - public DefaultParseFilter() { - this( - new org.zaproxy.zap.spider.SpiderParam(), - new ResourceBundle() { - - @Override - public Enumeration getKeys() { - return Collections.emptyEnumeration(); - } - - @Override - protected Object handleGetObject(String key) { - return ""; - } - }); - } - - /** - * Constructs a {@code DefaultParseFilter} with the given configurations and resource bundle. - * - *

The resource bundle is used to obtain the (internationalised) reasons of why the message - * was filtered. - * - * @param params the spider configurations - * @param resourceBundle the resource bundle to obtain the internationalised reasons. - * @throws IllegalArgumentException if any of the given parameters is {@code null}. - * @since 2.7.0 - */ - public DefaultParseFilter( - org.zaproxy.zap.spider.SpiderParam params, ResourceBundle resourceBundle) { - if (params == null) { - throw new IllegalArgumentException("Parameter params must not be null."); - } - if (resourceBundle == null) { - throw new IllegalArgumentException("Parameter resourceBundle must not be null."); - } - this.params = params; - - filterResultEmpty = - new FilterResult(resourceBundle.getString("spider.parsefilter.reason.empty")); - filterResultMaxSize = - new FilterResult(resourceBundle.getString("spider.parsefilter.reason.maxsize")); - filterResultNotText = - new FilterResult(resourceBundle.getString("spider.parsefilter.reason.nottext")); - } - - @Override - public FilterResult filtered(HttpMessage responseMessage) { - if (responseMessage == null - || responseMessage.getRequestHeader().isEmpty() - || responseMessage.getResponseHeader().isEmpty()) { - return filterResultEmpty; - } - - // if it's a file ending in "/.svn/entries", or "/.svn/wc.db", the SVN Entries or Git - // parsers will process it - // if it's a robots.txt or sitemap.xml the relevant parsers will process it - // regardless of type, and regardless of whether it exceeds the file size restriction below. - String fullfilename = responseMessage.getRequestHeader().getURI().getEscapedPath(); - if (fullfilename != null - && (SVN_SQLITE_FILENAME_PATTERN.matcher(fullfilename).find() - || SVN_XML_FILENAME_PATTERN.matcher(fullfilename).find() - || GIT_FILENAME_PATTERN.matcher(fullfilename).find() - || ROBOTS_FILENAME_PATTERN.matcher(fullfilename).find() - || SITEMAP_FILENAME_PATTERN.matcher(fullfilename).find())) { - return FilterResult.NOT_FILTERED; - } - - // Check response body size - if (responseMessage.getResponseBody().length() > params.getMaxParseSizeBytes()) { - getLogger() - .debug("Resource too large: {}", responseMessage.getRequestHeader().getURI()); - return filterResultMaxSize; - } - - // If it's a redirection, accept it, as the SpiderRedirectParser will process it - if (HttpStatusCode.isRedirection(responseMessage.getResponseHeader().getStatusCode())) { - return FilterResult.NOT_FILTERED; - } - - // Check response type. - if (!responseMessage.getResponseHeader().isText()) { - getLogger() - .debug("Resource is not text: {}", responseMessage.getRequestHeader().getURI()); - return filterResultNotText; - } - - return FilterResult.NOT_FILTERED; - } -} diff --git a/zap/src/main/java/org/zaproxy/zap/spider/filters/FetchFilter.java b/zap/src/main/java/org/zaproxy/zap/spider/filters/FetchFilter.java deleted file mode 100644 index 8c0ed5b67da..00000000000 --- a/zap/src/main/java/org/zaproxy/zap/spider/filters/FetchFilter.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2012 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.spider.filters; - -import org.apache.commons.httpclient.URI; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -/** - * A FetchFilter is used to filter which resources should be fetched and processed by the Spider and - * which shouldn't. This filter is applied before adding the resource in the processing queue. - * - * @deprecated (2.12.0) See the spider add-on in zap-extensions instead. - */ -@Deprecated -public abstract class FetchFilter { - - /** - * The FetchStatus enum is used as the status returned by a FetchFilter, stating if the uri is - * accepted or, if not, why was it not accepted. - */ - public enum FetchStatus { - /** The uri is VALID. */ - VALID, - /** The uri is VALID and is a seed. * */ - SEED, - /** The uri is out of context. */ - OUT_OF_CONTEXT, - /** The uri is out of scope. */ - OUT_OF_SCOPE, - /** The uri has an illegal protocol. */ - ILLEGAL_PROTOCOL, - /** The uri is skipped because of user rules. */ - USER_RULES - } - - /** - * The Constant log. - * - * @deprecated (2.10.0) Use {@link #getLogger()} instead. - */ - @Deprecated - protected static final org.apache.log4j.Logger log = - org.apache.log4j.Logger.getLogger(FetchFilter.class); - - private final Logger logger = LogManager.getLogger(getClass()); - - /** - * Gets the logger. - * - * @return the logger, never {@code null}. - * @since 2.10.0 - */ - protected Logger getLogger() { - return logger; - } - - /** - * Checks if the uri must be ignored and not processed and return the filter status. - * - * @param uri the uri to be processed - * @return the fetch status, stating if the uri is accepted or, if not, why was it not accepted. - */ - public abstract FetchStatus checkFilter(URI uri); -} diff --git a/zap/src/main/java/org/zaproxy/zap/spider/filters/HttpPrefixFetchFilter.java b/zap/src/main/java/org/zaproxy/zap/spider/filters/HttpPrefixFetchFilter.java deleted file mode 100644 index 4f326213877..00000000000 --- a/zap/src/main/java/org/zaproxy/zap/spider/filters/HttpPrefixFetchFilter.java +++ /dev/null @@ -1,335 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2016 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.spider.filters; - -import java.util.Arrays; -import java.util.Locale; -import org.apache.commons.httpclient.URI; -import org.apache.commons.httpclient.URIException; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -/** - * A {@code FetchFilter} that filters based on a HTTP or HTTPS {@code URI}. - * - *

The filtered {@code URI}s are required to start with the {@code URI} (the prefix) to be - * considered valid. - * - * @since 2.5.0 - * @see #checkFilter(URI) - * @deprecated (2.12.0) See the spider add-on in zap-extensions instead. - */ -@Deprecated -public class HttpPrefixFetchFilter extends FetchFilter { - - private static final Logger LOGGER = LogManager.getLogger(HttpPrefixFetchFilter.class); - - /** The normalised form of HTTP scheme, that is, all letters lowercase. */ - private static final String HTTP_SCHEME = "http"; - - /** The normalised form of HTTPS scheme, that is, all letters lowercase. */ - private static final String HTTPS_SCHEME = "https"; - - /** The port number that indicates that a port is the default of a scheme. */ - private static final int DEFAULT_PORT = -1; - - /** - * The port number that indicates that a port is of an unknown scheme (that is, non HTTP and - * HTTPS). - */ - private static final int UNKNOWN_PORT = -2; - - /** The default port number of HTTP scheme. */ - private static final int DEFAULT_HTTP_PORT = 80; - - /** The default port number of HTTPS scheme. */ - private static final int DEFAULT_HTTPS_PORT = 443; - - /** The scheme used for filtering. Never {@code null}. */ - private final String scheme; - - /** The host used for filtering. Never {@code null}. */ - private final String host; - - /** The port used for filtering. */ - private final int port; - - /** The path used for filtering. Might be {@code null}. */ - private final char[] path; - - /** - * Constructs a {@code HttpPrefixFetchFilter} using the given {@code URI} as prefix. - * - *

The user info, query component and fragment of the given {@code URI} are discarded. The - * scheme and domain comparisons are done in a case insensitive way while the path component - * comparison is case sensitive. - * - * @param prefix the {@code URI} that will be used as prefix - * @throws IllegalArgumentException if any of the following conditions is {@code true}: - *

    - *
  • The given {@code prefix} is {@code null}; - *
  • The given {@code prefix} has {@code null} scheme; - *
  • The scheme of the given {@code prefix} is not HTTP or HTTPS; - *
  • The given {@code prefix} has {@code null} host; - *
  • The given {@code prefix} has malformed host. - *
- */ - public HttpPrefixFetchFilter(URI prefix) { - if (prefix == null) { - throw new IllegalArgumentException("Parameter prefix must not be null."); - } - - char[] rawScheme = prefix.getRawScheme(); - if (rawScheme == null) { - throw new IllegalArgumentException("Parameter prefix must have a scheme."); - } - String normalisedScheme = normalisedScheme(rawScheme); - if (!isHttpOrHttps(normalisedScheme)) { - throw new IllegalArgumentException("The prefix's scheme must be HTTP or HTTPS."); - } - scheme = normalisedScheme; - - if (prefix.getRawHost() == null) { - throw new IllegalArgumentException("Parameter prefix must have a host."); - } - try { - host = normalisedHost(prefix); - } catch (URIException e) { - throw new IllegalArgumentException("Failed to obtain the host from the prefix:", e); - } - - port = normalisedPort(scheme, prefix.getPort()); - path = prefix.getRawPath(); - } - - /** - * Returns the normalised form of the given {@code scheme}. - * - *

The normalisation process consists in converting the scheme to lowercase, if {@code null} - * it is returned an empty {@code String}. - * - * @param scheme the scheme that will be normalised - * @return a {@code String} with the host scheme, never {@code null} - * @see URI#getRawScheme() - */ - private static String normalisedScheme(char[] scheme) { - if (scheme == null) { - return ""; - } - return new String(scheme).toLowerCase(Locale.ROOT); - } - - /** - * Tells whether or not the given {@code scheme} is HTTP or HTTPS. - * - * @param scheme the normalised scheme, might be {@code null} - * @return {@code true} if the {@code scheme} is HTTP or HTTPS, {@code false} otherwise - */ - private static boolean isHttpOrHttps(String scheme) { - return isHttp(scheme) || isHttps(scheme); - } - - /** - * Tells whether or not the given {@code scheme} is HTTP. - * - * @param scheme the normalised scheme, might be {@code null} - * @return {@code true} if the {@code scheme} is HTTP, {@code false} otherwise - */ - private static boolean isHttp(String scheme) { - return HTTP_SCHEME.equals(scheme); - } - - /** - * Tells whether or not the given {@code scheme} is HTTPS. - * - * @param scheme the normalised scheme, might be {@code null} - * @return {@code true} if the {@code scheme} is HTTPS, {@code false} otherwise - */ - private static boolean isHttps(String scheme) { - return HTTPS_SCHEME.equals(scheme); - } - - /** - * Returns the normalised form of the host of the given {@code uri}. - * - *

The normalisation process consists in converting the host to lowercase, if {@code null} it - * is returned an empty {@code String}. - * - * @param uri the URI whose host will be extracted and normalised - * @return a {@code String} with the host normalised, never {@code null} - * @throws URIException if the host of the given {@code uri} is malformed - */ - private static String normalisedHost(URI uri) throws URIException { - if (uri.getRawHost() == null) { - return ""; - } - return uri.getHost().toLowerCase(Locale.ROOT); - } - - /** - * Returns the normalised form of the given {@code port}, based on the given {@code scheme}. - * - *

If the port is non-default (as given by {@link #DEFAULT_PORT}), it's immediately returned. - * Otherwise, for schemes HTTP and HTTPS it's returned 80 and 443, respectively, for any other - * scheme it's returned {@link #UNKNOWN_PORT}. - * - * @param scheme the (normalised) scheme of the URI where the port was defined - * @param port the port to normalise - * @return the normalised port - * @see #normalisedScheme(char[]) - * @see URI#getPort() - */ - private static int normalisedPort(String scheme, int port) { - if (port != DEFAULT_PORT) { - return port; - } - - if (isHttp(scheme)) { - return DEFAULT_HTTP_PORT; - } - - if (isHttps(scheme)) { - return DEFAULT_HTTPS_PORT; - } - - return UNKNOWN_PORT; - } - - /** - * Gets the prefix normalised, as it is used to filter the {@code URI}s. - * - * @return a {@code String} with the prefix normalised - * @see #checkFilter(URI) - */ - public String getNormalisedPrefix() { - StringBuilder strBuilder = new StringBuilder(); - strBuilder.append(scheme).append("://").append(host); - if (!isDefaultHttpOrHttpsPort(scheme, port)) { - strBuilder.append(':').append(port); - } - if (path != null) { - strBuilder.append(path); - } - return strBuilder.toString(); - } - - /** - * Tells whether or not the given {@code port} is the default for the given {@code scheme}. - * - *

The method returns always {@code false} for non HTTP or HTTPS schemes. - * - * @param scheme the scheme of a URI, might be {@code null} - * @param port the port of a URI - * @return {@code true} if the {@code port} is the default for the given {@code scheme}, {@code - * false} otherwise - */ - private static boolean isDefaultHttpOrHttpsPort(String scheme, int port) { - if (port == DEFAULT_HTTP_PORT && isHttp(scheme)) { - return true; - } - if (port == DEFAULT_HTTPS_PORT && isHttps(scheme)) { - return true; - } - return false; - } - - /** - * Filters any URI that does not start with the defined prefix. - * - * @return {@code FetchStatus.VALID} if the {@code uri} starts with the {@code prefix}, {@code - * FetchStatus.OUT_OF_SCOPE} otherwise - */ - @Override - public FetchStatus checkFilter(URI uri) { - if (uri == null) { - return FetchStatus.OUT_OF_SCOPE; - } - - String otherScheme = normalisedScheme(uri.getRawScheme()); - if (port != normalisedPort(otherScheme, uri.getPort())) { - return FetchStatus.OUT_OF_SCOPE; - } - - if (!scheme.equals(otherScheme)) { - return FetchStatus.OUT_OF_SCOPE; - } - - if (!hasSameHost(uri)) { - return FetchStatus.OUT_OF_SCOPE; - } - - if (!startsWith(uri.getRawPath(), path)) { - return FetchStatus.OUT_OF_SCOPE; - } - - return FetchStatus.VALID; - } - - /** - * Tells whether or not the given {@code uri} has the same host as required by this prefix. - * - *

For malformed hosts it returns always {@code false}. - * - * @param uri the {@code URI} whose host will be checked - * @return {@code true} if the host is same, {@code false} otherwise - */ - private boolean hasSameHost(URI uri) { - try { - return host.equals(normalisedHost(uri)); - } catch (URIException e) { - LOGGER.warn("Failed to normalise host: {}", Arrays.toString(uri.getRawHost()), e); - } - return false; - } - - /** - * Tells whether or not the given {@code array} starts with the given {@code prefix}. - * - *

The {@code prefix} might be {@code null} in which case it's considered that the {@code - * array} starts with the prefix. - * - * @param array the array that will be tested if starts with the prefix, might be {@code null} - * @param prefix the array used as prefix, might be {@code null} - * @return {@code true} if the {@code array} starts with the {@code prefix}, {@code false} - * otherwise - */ - private static boolean startsWith(char[] array, char[] prefix) { - if (prefix == null) { - return true; - } - - if (array == null) { - return false; - } - - int length = prefix.length; - if (array.length < length) { - return false; - } - - for (int i = 0; i < length; i++) { - if (prefix[i] != array[i]) { - return false; - } - } - - return true; - } -} diff --git a/zap/src/main/java/org/zaproxy/zap/spider/filters/MaxChildrenFetchFilter.java b/zap/src/main/java/org/zaproxy/zap/spider/filters/MaxChildrenFetchFilter.java deleted file mode 100644 index 5cb9efbc65e..00000000000 --- a/zap/src/main/java/org/zaproxy/zap/spider/filters/MaxChildrenFetchFilter.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2014 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.spider.filters; - -import org.apache.commons.httpclient.URI; -import org.parosproxy.paros.model.Model; -import org.parosproxy.paros.model.SiteNode; - -/** - * The MaxChildrenFetchFilter defines a filter rule for limiting the number of children explored. - * - * @deprecated (2.12.0) See the spider add-on in zap-extensions instead. - */ -@Deprecated -public class MaxChildrenFetchFilter extends FetchFilter { - - private int maxChildren = -1; - - private Model model; - - @Override - public FetchStatus checkFilter(URI uri) { - getLogger().debug("Checking: {}", uri); - - SiteNode parent = model.getSession().getSiteTree().findClosestParent(uri); - if (parent != null) { - if (maxChildren > 0 && parent.getChildCount() > maxChildren) { - return FetchStatus.USER_RULES; - } - } - - return FetchStatus.VALID; - } - - public void setMaxChildren(int maxChildren) { - this.maxChildren = maxChildren; - } - - /** - * Sets the model - * - * @param model the model used to check the number of children of a node - */ - public void setModel(Model model) { - this.model = model; - } -} diff --git a/zap/src/main/java/org/zaproxy/zap/spider/filters/MaxChildrenParseFilter.java b/zap/src/main/java/org/zaproxy/zap/spider/filters/MaxChildrenParseFilter.java deleted file mode 100644 index c9dfd3073de..00000000000 --- a/zap/src/main/java/org/zaproxy/zap/spider/filters/MaxChildrenParseFilter.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2014 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.spider.filters; - -import java.util.Collections; -import java.util.Enumeration; -import java.util.ResourceBundle; -import org.parosproxy.paros.model.Model; -import org.parosproxy.paros.model.SiteNode; -import org.parosproxy.paros.network.HttpMessage; - -/** - * The MaxChildrenFetchFilter defines a filter rule for limiting the number of children explored. - * - * @deprecated (2.12.0) See the spider add-on in zap-extensions instead. - */ -@Deprecated -public class MaxChildrenParseFilter extends ParseFilter { - - private int maxChildren = -1; - - private Model model; - - private final FilterResult filtered; - - /** - * Constructs a {@code MaxChildrenParseFilter}, with no reason of why the message was filtered. - * - * @deprecated (2.7.0) Use {@link #MaxChildrenParseFilter(ResourceBundle)} instead. - */ - @Deprecated - public MaxChildrenParseFilter() { - this( - new ResourceBundle() { - - @Override - public Enumeration getKeys() { - return Collections.emptyEnumeration(); - } - - @Override - protected Object handleGetObject(String key) { - return ""; - } - }); - } - - /** - * Constructs a {@code MaxChildrenParseFilter} with the given resource bundle. - * - *

The resource bundle is used to obtain the (internationalised) reason of why the message - * was filtered. - * - * @param resourceBundle the resource bundle to obtain the internationalised reason. - * @throws IllegalArgumentException if the given parameter is {@code null}. - * @since 2.7.0 - */ - public MaxChildrenParseFilter(ResourceBundle resourceBundle) { - if (resourceBundle == null) { - throw new IllegalArgumentException("Parameter resourceBundle must not be null."); - } - filtered = - new FilterResult(resourceBundle.getString("spider.parsefilter.reason.maxchildren")); - } - - @Override - public FilterResult filtered(HttpMessage responseMessage) { - - SiteNode parent = model.getSession().getSiteTree().findClosestParent(responseMessage); - if (parent != null) { - if (maxChildren > 0 && parent.getChildCount() > maxChildren) { - return filtered; - } - } - - return FilterResult.NOT_FILTERED; - } - - public void setMaxChildren(int maxChildren) { - this.maxChildren = maxChildren; - } - - /** - * Sets the model - * - * @param model the model used to check the number of children of a node - */ - public void setModel(Model model) { - this.model = model; - } -} diff --git a/zap/src/main/java/org/zaproxy/zap/spider/filters/ParseFilter.java b/zap/src/main/java/org/zaproxy/zap/spider/filters/ParseFilter.java deleted file mode 100644 index da8c794a31b..00000000000 --- a/zap/src/main/java/org/zaproxy/zap/spider/filters/ParseFilter.java +++ /dev/null @@ -1,160 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2012 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.spider.filters; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.parosproxy.paros.network.HttpMessage; - -/** - * A ParseFilter is used to filter which resources should be parsed by the Spider after they have - * already been fetched and which shouldn't. - * - * @deprecated (2.12.0) See the spider add-on in zap-extensions instead. - */ -@Deprecated -public abstract class ParseFilter { - - /** - * The Constant log. - * - * @deprecated (2.10.0) Use {@link #getLogger()} instead. - */ - @Deprecated - protected static final org.apache.log4j.Logger log = - org.apache.log4j.Logger.getLogger(ParseFilter.class); - - private final Logger logger = LogManager.getLogger(getClass()); - - /** - * Gets the logger. - * - * @return the logger, never {@code null}. - * @since 2.10.0 - */ - protected Logger getLogger() { - return logger; - } - - /** - * Checks if the resource must be ignored and not processed. - * - * @param responseMessage the response message after the resource was fetched - * @return true, if is filtered - * @deprecated (2.7.0) Use {@link #filtered(HttpMessage)} instead, which allows to provide the - * reason why the message was filtered. - */ - @Deprecated - public boolean isFiltered(HttpMessage responseMessage) { - return filtered(responseMessage).isFiltered(); - } - - /** - * Tells whether or not the given resource is filtered. Filtered resources are not parsed. - * - *

Default is not filtered. - * - * @param responseMessage the HTTP message containing the response to be or not parsed. - * @return the filter result, must not be {@code null}. - * @since 2.7.0 - */ - public FilterResult filtered(HttpMessage responseMessage) { - return FilterResult.NOT_FILTERED; - } - - /** - * The result of a {@link ParseFilter}'s check. - * - *

Used to indicate if a resource was filtered and why. - * - * @since 2.7.0 - * @see #NOT_FILTERED - * @see #FilterResult(String) - */ - public static final class FilterResult { - - /** Indicates that the resource was not filtered. */ - public static final FilterResult NOT_FILTERED = new FilterResult(); - - /** - * Indicates that the resource is wanted by a custom parser. The resource skips the default - * filter, if no other filter filters it out. - */ - public static final FilterResult WANTED = new FilterResult(); - - /** Indicates that the resource was filtered, with no specific reason. */ - public static final FilterResult FILTERED = new FilterResult(""); - - private final boolean filtered; - private final String reason; - - /** Constructs a {@code FilterResult}, not filtered and with empty reason. */ - private FilterResult() { - this(false, ""); - } - - /** - * Constructs a {@code FilterResult} with the reason why the resource will not be parsed. - * - * @param reason the reason why the resource was filtered. - * @see #NOT_FILTERED - */ - public FilterResult(String reason) { - this(true, reason); - } - - /** - * Constructs a {@code FilterResult} with the given filtered state and reason. - * - * @param filtered {@code true} if the resource was filtered, {@code false} otherwise. - * @param reason the reason why the resource was filtered. - * @throws IllegalArgumentException if the given {@code reason} is {@code null}. - */ - private FilterResult(boolean filtered, String reason) { - if (reason == null) { - throw new IllegalArgumentException("Parameter reason must not be null."); - } - this.filtered = filtered; - this.reason = reason; - } - - /** - * Tells whether or not the resource was filtered. - * - *

Filtered resources are not parsed. - * - * @return {@code true} if the resource was filtered, {@code false} otherwise. - * @see #getReason() - */ - public boolean isFiltered() { - return filtered; - } - - /** - * Gets the reason why the resource was filtered. - * - * @return the reason why the resource was filtered, never {@code null}. - * @see #isFiltered() - */ - public String getReason() { - return reason; - } - } -} diff --git a/zap/src/main/java/org/zaproxy/zap/spider/parser/SpiderGitParser.java b/zap/src/main/java/org/zaproxy/zap/spider/parser/SpiderGitParser.java deleted file mode 100644 index f21960cda0b..00000000000 --- a/zap/src/main/java/org/zaproxy/zap/spider/parser/SpiderGitParser.java +++ /dev/null @@ -1,331 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2014 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.spider.parser; - -import java.nio.ByteBuffer; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import net.htmlparser.jericho.Source; -import org.parosproxy.paros.network.HttpMessage; - -/** - * The Class SpiderGitParser is used for parsing Git metadata from the .git/index file This parser - * currently supports Git internal index file versions 2,3, and 4. It does not currently support - * version 1, since this version is no longer supported. Version 1 appears to have disappeared with - * Git version 0.05 in 2005. - * - * @author 70pointer - * @deprecated (2.12.0) See the spider add-on in zap-extensions instead. - */ -@Deprecated -public class SpiderGitParser extends SpiderParser { - - /** a pattern to match the file name of the Git index file */ - private static final Pattern gitIndexFilenamePattern = Pattern.compile("/.git/index$"); - - /** a pattern to match the content of the Git index file */ - private static final Pattern gitIndexContentPattern = Pattern.compile("^DIRC"); - - private Pattern GIT_FILE_PATTERN = Pattern.compile("/\\.git/index$"); - - /** - * Instantiates a new spider Git Index parser. - * - * @param params the params - * @throws NullPointerException if {@code params} is null. - */ - public SpiderGitParser(org.zaproxy.zap.spider.SpiderParam params) { - super(params); - } - - @SuppressWarnings("unused") - @Override - public boolean parseResource(HttpMessage message, Source source, int depth) { - - // parse the Git index file, based on publicly available (but incomplete) documentation of - // the file format, and some reverse-engineering. - if (message == null || !getSpiderParam().isParseGit()) { - return false; - } - getLogger().debug("Parsing a Git resource..."); - - // Get the response content - byte[] data = message.getResponseBody().getBytes(); - String baseURL = message.getRequestHeader().getURI().toString(); - - try { - String fullpath = message.getRequestHeader().getURI().getPath(); - if (fullpath == null) fullpath = ""; - getLogger().debug("The full path is [{}]", fullpath); - - // make sure the file name is as expected - Matcher gitIndexFilenameMatcher = gitIndexFilenamePattern.matcher(fullpath); - if (!gitIndexFilenameMatcher.find()) { - getLogger().warn("This path cannot be handled by the Git parser: {}", fullpath); - return false; - } - - // dealing with the Git Index file - - Matcher gitIndexContentMatcher = gitIndexContentPattern.matcher(new String(data)); - if (!gitIndexContentMatcher.find()) { - getLogger() - .debug( - "The file '{}' could not be parsed as a Git Index file due to unexpected content", - fullpath); - return false; - } - // it looks like a duck, and quacks like a duck. - // although it could still be an animatronic duck - - ByteBuffer dataBuffer = ByteBuffer.wrap(data); - - byte[] dircArray = new byte[4]; - dataBuffer.get(dircArray, 0, 4); - - int indexFileVersion = dataBuffer.getInt(); - getLogger().debug("The Git index file version is {}", indexFileVersion); - - int indexEntryCount = dataBuffer.getInt(); - getLogger().debug("{} entries were found in the Git index file ", indexEntryCount); - - if (indexFileVersion != 2 && indexFileVersion != 3 && indexFileVersion != 4) { - throw new Exception( - "Only Git Index File versions 2, 3, and 4 are currently supported. Git Index File Version " - + indexFileVersion - + " was found."); - } - - // for version 4 (and upwards?), we need to know the previous entry name, so store it - String previousIndexEntryName = ""; - for (int entryIndex = 0; entryIndex < indexEntryCount; entryIndex++) { - int entryBytesRead = 0; - int indexEntryCtime1 = dataBuffer.getInt(); - entryBytesRead += 4; - getLogger().debug("Entry {} has indexEntryCtime1 {}", entryIndex, indexEntryCtime1); - int indexEntryCtime2 = dataBuffer.getInt(); - entryBytesRead += 4; - int indexEntryMtime1 = dataBuffer.getInt(); - entryBytesRead += 4; - int indexEntryMtime2 = dataBuffer.getInt(); - entryBytesRead += 4; - int indexEntryDev = dataBuffer.getInt(); - entryBytesRead += 4; - int indexEntryInode = dataBuffer.getInt(); - entryBytesRead += 4; - int indexEntryMode = dataBuffer.getInt(); - entryBytesRead += 4; - int indexEntryUid = dataBuffer.getInt(); - entryBytesRead += 4; - int indexEntryGid = dataBuffer.getInt(); - entryBytesRead += 4; - int indexEntrySize = dataBuffer.getInt(); - entryBytesRead += 4; - getLogger().debug("Entry {} has size {}", entryIndex, indexEntrySize); - - // size is unspecified for the entry id, but it seems to be 40 bytes SHA-1 string - // stored as 20 bytes, network order - byte[] indexEntryIdBuffer = new byte[20]; - dataBuffer.get(indexEntryIdBuffer, 0, 20); - entryBytesRead += 20; - String indexEntryId = new String(indexEntryIdBuffer); - - short indexEntryFlags = dataBuffer.getShort(); - entryBytesRead += 2; - getLogger().debug("Entry {} has flags {}", entryIndex, indexEntryFlags); - - // mask off all but the least significant 12 bits of the index entry flags to get - // the length of the name in bytes - int indexEntryNameByteLength = indexEntryFlags & 4095; - getLogger() - .debug( - "Entry {} has a name of length {}", - entryIndex, - indexEntryNameByteLength); - - // mask off all but the second most significant 12 bit of the index entry flags to - // get the extended flag for the entry - // int indexEntryExtendedFlag = indexEntryFlags & (int)16384; - int indexEntryExtendedFlag = ((indexEntryFlags & (1 << 14)) >> 14); - getLogger() - .debug( - "Entry {} has an extended flag of {}", - entryIndex, - indexEntryExtendedFlag); - - // check that we parsed out the index entry extended flag correctly. - // this is more of an assertion than anything. It's already saved my bacon once. - if (indexEntryExtendedFlag != 0 && indexEntryExtendedFlag != 1) { - throw new Exception( - "Error parsing out the extended flag for index entry " - + entryIndex - + ". We got " - + indexEntryExtendedFlag); - } - if (indexFileVersion == 2 && indexEntryExtendedFlag != 0) { - throw new Exception( - "Index File Version 2 is supposed to have the extended flag set to 0. For index entry " - + entryIndex - + ", it is set to " - + indexEntryExtendedFlag); - } - - // specific to version 3 and above, if the extended flag is set for the entry. - if (indexFileVersion > 2 && indexEntryExtendedFlag == 1) { - getLogger() - .debug( - "For Index file version {}, reading an extra 16 bits for Entry {}", - indexFileVersion, - entryIndex); - short indexEntryExtendedFlags = dataBuffer.getShort(); - entryBytesRead += 2; - getLogger() - .debug( - "Entry {} has (optional) extended flags {}", - entryIndex, - indexEntryExtendedFlags); - } - - String indexEntryName = null; - if (indexFileVersion > 3) { - getLogger() - .debug( - "Inflating the (deflated) entry name for index entry {} based on the previous entry name, since Index file version {} requires this", - entryIndex, - indexFileVersion); - - // get bytes until we find one with the msb NOT set. count the bytes. - int n = 0, removeNfromPreviousName = 0; - byte msbsetmask = (byte) (1 << 7); // 1000 0000 - byte msbunsetmask = (byte) ((~msbsetmask) & 0xFF); // 0111 1111 - while (++n > 0) { - byte byteRead = dataBuffer.get(); - entryBytesRead++; - if (n == 1) // zero the msb of the first byte read - removeNfromPreviousName = - (removeNfromPreviousName << 8) - | (0xFF & (byteRead & msbunsetmask)); - else // set the msb of subsequent bytes read - removeNfromPreviousName = - (removeNfromPreviousName << 8) - | (0xFF & (byteRead | msbsetmask)); - if ((byteRead & msbsetmask) == 0) - break; // break if msb is NOT set in the byte - } - - getLogger() - .debug( - "We read {} bytes of variable length data from before the start of the entry name", - n); - if (n > 4) - throw new Exception( - "An entry name is never expected to be > 2^^32 bytes long. Some file corruption may have occurred, or a parsing error has occurred"); - - // now read the (partial) name for the current entry - int bytesToReadCurrentNameEntry = - indexEntryNameByteLength - - (previousIndexEntryName.length() - removeNfromPreviousName); - byte[] indexEntryNameBuffer = new byte[bytesToReadCurrentNameEntry]; - dataBuffer.get(indexEntryNameBuffer, 0, bytesToReadCurrentNameEntry); - entryBytesRead += bytesToReadCurrentNameEntry; - - // build it up - indexEntryName = - previousIndexEntryName.substring( - 0, - previousIndexEntryName.length() - - removeNfromPreviousName) - + new String(indexEntryNameBuffer); - } else { - // indexFileVersion <= 3 (waaaaay simpler logic, but the index file is larger in - // this version than for v4+) - byte[] indexEntryNameBuffer = new byte[indexEntryNameByteLength]; - dataBuffer.get(indexEntryNameBuffer, 0, indexEntryNameByteLength); - entryBytesRead += indexEntryNameByteLength; - indexEntryName = new String(indexEntryNameBuffer); - } - - getLogger().debug("Entry {} has name {}", entryIndex, indexEntryName); - - // and store off the index entry name, for the next iteration - previousIndexEntryName = indexEntryName; - // skip past the zero byte terminating the string (whose purpose seems completely - // pointless to me, but hey) - byte indexEntryNul = dataBuffer.get(); - entryBytesRead++; - - // the padding after the pathname does not exist for versions 4 or later. - if (indexFileVersion < 4) { - getLogger() - .debug( - "Aligning to an 8 byte boundary after Entry {}, since Index file version {} mandates 64 bit alignment for index entries", - entryIndex, - indexFileVersion); - - int entryBytesToRead = ((8 - (entryBytesRead % 8)) % 8); - getLogger() - .debug( - "The number of bytes read for index entry {} thus far is: {}", - entryIndex, - entryBytesRead); - getLogger() - .debug( - "So we must read {} bytes to stay on a 64 bit boundary", - entryBytesToRead); - // read the 0-7 (NUL) bytes to keep reading index entries on an 8 byte boundary - byte[] indexEntryPadBuffer = new byte[entryBytesToRead]; - dataBuffer.get(indexEntryPadBuffer, 0, entryBytesToRead); - entryBytesRead += entryBytesToRead; - } else { - getLogger() - .debug( - "Not aligning to an 8 byte boundary after Entry {}, since Index file version {} does not mandate 64 bit alignment for index entries", - entryIndex, - indexFileVersion); - } - - // Git does not store entries for directories, but just files/symlinks/Git links, so - // no need to handle directories here, unlike with SVN, for instance. - if (indexEntryName != null && indexEntryName.length() > 0) { - getLogger() - .info( - "Found file/symbolic link/gitlink {} in the Git entries file", - indexEntryName); - processURL(message, depth, "../" + indexEntryName, baseURL); - } - } - // all good, we're outta here. - // We consider the message fully parsed, so it doesn't get parsed by 'fallback' parsers - return true; - - } catch (Exception e) { - getLogger().warn("An error occurred trying to parse Git url '{}': ", baseURL, e); - // We consider the message fully parsed, so it doesn't get parsed by 'fallback' parsers - return true; - } - } - - @Override - public boolean canParseResource(HttpMessage message, String path, boolean wasAlreadyParsed) { - // matches the file name of files that should be parsed with the GIT file parser - Matcher matcher = GIT_FILE_PATTERN.matcher(path); - return matcher.find(); - } -} diff --git a/zap/src/main/java/org/zaproxy/zap/spider/parser/SpiderHtmlFormParser.java b/zap/src/main/java/org/zaproxy/zap/spider/parser/SpiderHtmlFormParser.java deleted file mode 100644 index 62d803aa2a9..00000000000 --- a/zap/src/main/java/org/zaproxy/zap/spider/parser/SpiderHtmlFormParser.java +++ /dev/null @@ -1,634 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2012 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.spider.parser; - -import java.io.UnsupportedEncodingException; -import java.net.URLEncoder; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.SortedSet; -import java.util.TreeSet; -import java.util.stream.Collectors; -import net.htmlparser.jericho.Attribute; -import net.htmlparser.jericho.Element; -import net.htmlparser.jericho.FormControl; -import net.htmlparser.jericho.FormField; -import net.htmlparser.jericho.FormFields; -import net.htmlparser.jericho.HTMLElementName; -import net.htmlparser.jericho.Segment; -import net.htmlparser.jericho.Source; -import org.apache.commons.httpclient.URI; -import org.apache.commons.lang3.StringUtils; -import org.parosproxy.paros.network.HttpMessage; - -/** - * The Class SpiderHtmlFormParser is used for parsing HTML files for processing forms. - * - * @deprecated (2.12.0) See the spider add-on in zap-extensions instead. - */ -@Deprecated -@SuppressWarnings("removal") -public class SpiderHtmlFormParser extends SpiderParser { - - private static final String ENCODING_TYPE = "UTF-8"; - private static final String DEFAULT_EMPTY_VALUE = ""; - private static final String METHOD_GET = "GET"; - private static final String METHOD_POST = "POST"; - private URI uri; - private String url; - - /** The form attributes */ - private Map envAttributes = new HashMap<>(); - - /** Create new Value Generator field */ - private final org.zaproxy.zap.model.ValueGenerator valueGenerator; - - /** - * Instantiates a new spider html form parser. - * - * @param param the parameters for the spider - * @throws IllegalArgumentException if {@code param} is null. - */ - public SpiderHtmlFormParser(org.zaproxy.zap.spider.SpiderParam param) { - this(param, new org.zaproxy.zap.model.DefaultValueGenerator()); - } - - /** - * Instantiates a new spider html form parser. - * - * @param param the parameters for the spider - * @param valueGenerator the ValueGenerator - * @throws IllegalArgumentException if {@code valueGenerator} is null. - * @throws NullPointerException if {@code param} is null. - */ - public SpiderHtmlFormParser( - org.zaproxy.zap.spider.SpiderParam param, - org.zaproxy.zap.model.ValueGenerator valueGenerator) { - super(param); - if (valueGenerator == null) { - throw new IllegalArgumentException("Parameter valueGenerator must not be null."); - } - this.valueGenerator = valueGenerator; - } - - @Override - public boolean parseResource(HttpMessage message, Source source, int depth) { - getLogger().debug("Parsing an HTML message for forms..."); - // If form processing is disabled, don't parse anything - if (!getSpiderParam().isProcessForm()) { - return false; - } - - // Prepare the source, if not provided - if (source == null) { - source = new Source(message.getResponseBody().toString()); - } - - // Get the context (base url) - String baseURL = message.getRequestHeader().getURI().toString(); - uri = message.getRequestHeader().getURI(); - - // Try to see if there's any BASE tag that could change the base URL - Element base = source.getFirstElement(HTMLElementName.BASE); - if (base != null) { - getLogger().debug("Base tag was found in HTML: {}", base.getDebugInfo()); - String href = base.getAttributeValue("href"); - if (href != null && !href.isEmpty()) { - baseURL = getCanonicalURL(href, baseURL); - } - } - - // Go through the forms - List forms = source.getAllElements(HTMLElementName.FORM); - - for (Element form : forms) { - // Clear the attributes for each form and store their key and values - envAttributes.clear(); - for (Attribute att : form.getAttributes()) { - envAttributes.put(att.getKey(), att.getValue()); - } - // Get method and action - String formMethod = form.getAttributeValue("method"); - - // A single form can have multiple actions associated to it - List formActions = processFormActions(form, formMethod, baseURL, source); - - for (FormAction fAction : formActions) { - String action = fAction.action; - String method = fAction.method; - getLogger() - .debug("Found new form with method: '{}' and action: {}", method, action); - - // If POSTing forms is not enabled, skip processing of forms with POST method - if (!getSpiderParam().isPostForm() - && method != null - && method.trim().equalsIgnoreCase(METHOD_POST)) { - getLogger().debug("Skipping form with POST method because of user settings."); - continue; - } - - // Clear the fragment, if any, as it does not have any relevance for the server - if (action.contains("#")) { - int fs = action.lastIndexOf("#"); - action = action.substring(0, fs); - } - - url = getCanonicalURL(action, baseURL); - FormData formData = prepareFormDataSet(source, form); - - // Process the case of a POST method - if (method != null && method.trim().equalsIgnoreCase(METHOD_POST)) { - // Build the absolute canonical URL - String fullURL = getCanonicalURL(action, baseURL); - if (fullURL == null) { - return false; - } - getLogger().debug("Canonical URL constructed using '{}: {}", action, fullURL); - - /* - * Ignore encoding, as we will not POST files anyway, so using - * "application/x-www-form-urlencoded" is adequate - */ - // String encoding = form.getAttributeValue("enctype"); - // if (encoding != null && encoding.equals("multipart/form-data")) - - for (String submitData : formData) { - notifyPostResourceFound(message, depth, fullURL, submitData); - } - - } // Process anything else as a GET method - else { - - // Process the final URL - if (action.contains("?")) { - if (action.endsWith("?")) { - processGetForm(message, depth, action, baseURL, formData); - } else { - processGetForm(message, depth, action + "&", baseURL, formData); - } - } else { - processGetForm(message, depth, action + "?", baseURL, formData); - } - } - } - } - - return false; - } - - private static class FormAction { - final String action; - final String method; - - FormAction(String action, String method) { - this.action = action; - this.method = method; - } - } - - /** - * Processes the given form element into, possibly, several URLs. - * - *

For each button present in the form, we are returning the corresponding action or - * formaction attribute defining the URL that should handle the form submission - * - * @param form the form to inspect - * @param baseURL the base URL - * @return a list of FormAction objects containing the action and associated method - */ - private List processFormActions( - Element form, String originalMethod, String baseURL, Source source) { - List formActions = new ArrayList<>(); - - String action = form.getAttributeValue("action"); - // If no action, use the base url - if (action == null) { - getLogger().debug("No form 'action' defined. Using base URL: {}", baseURL); - action = baseURL; - } - - // Check for possible formaction attributes in child elements with button tag. - // formaction attributes override the action of the parent form - - // Find direct child of the form element first - List formButtonElements = - form.getChildElements().stream() - .filter(this::allowedButtonType) - .filter(element -> StringUtils.isEmpty(element.getAttributeValue("form"))) - .collect(Collectors.toList()); - - // The form has an id, identify possible associated buttons anywhere in the document - if (StringUtils.isNotEmpty(form.getAttributeValue("id"))) { - String targetId = form.getAttributeValue("id"); - formButtonElements.addAll( - source.getAllElements(HTMLElementName.BUTTON).stream() - .filter(this::allowedButtonType) - .filter( - element -> - StringUtils.equals( - element.getAttributeValue("form"), targetId)) - .collect(Collectors.toList())); - } - - if (!formButtonElements.isEmpty()) { - final String defaultAction = action; - formButtonElements.forEach( - button -> { - // A button without a formaction submits to the default action for the form - if (StringUtils.isEmpty(button.getAttributeValue("formaction"))) { - formActions.add( - new FormAction( - defaultAction, - processFormMethodWithButton(button, originalMethod))); - } else { - formActions.add( - new FormAction( - button.getAttributeValue("formaction"), - processFormMethodWithButton(button, originalMethod))); - } - }); - } else { - // If no buttons, return just the default action for the form - formActions.add(new FormAction(action, originalMethod)); - } - - return formActions; - } - - /** - * Function that defines the allowed button types that should be considered in a form, reset and - * button type buttons do not trigger any action so should be ignored - * - * @param element an Element of type button - * @return true if the button should be processed - */ - private boolean allowedButtonType(Element element) { - String type = element.getAttributeValue("type"); - return element.getStartTag().getName().equals(HTMLElementName.BUTTON) - && !StringUtils.equalsIgnoreCase(type, "button") - && !StringUtils.equalsIgnoreCase(type, "reset"); - } - - /** - * Defines the correct form method to use in case a button belongs to a form. Valid form methods - * are GET and POST, if a button has no valid form method the original form method is returned - * - * @param button an Element of type button - * @param defaultMethod the default method of the parent form - * @return the form method to be used - */ - private String processFormMethodWithButton(Element button, String defaultMethod) { - String buttonMethod = button.getAttributeValue("formmethod"); - return StringUtils.isEmpty(buttonMethod) - || (!buttonMethod.equalsIgnoreCase(METHOD_GET) - && !buttonMethod.equalsIgnoreCase(METHOD_POST)) - ? defaultMethod - : buttonMethod; - } - - /** - * Processes the given GET form data into, possibly, several URLs. - * - *

For each submit field present in the form data is processed one URL, which includes - * remaining normal fields. - * - * @param message the source message - * @param depth the current depth - * @param action the action - * @param baseURL the base URL - * @param formData the GET form data - * @see #processURL(HttpMessage, int, String, String) - */ - private void processGetForm( - HttpMessage message, int depth, String action, String baseURL, FormData formData) { - for (String submitData : formData) { - getLogger() - .debug( - "Submitting form with GET method and query with form parameters: {}", - submitData); - processURL(message, depth, action + submitData, baseURL); - } - } - - /** - * Prepares the form data set. A form data set is a sequence of control-name/current-value pairs - * constructed from successful controls, which will be sent with a GET/POST request for a form. - * - * @see HTML 4.01 - * Specification - 17.13.3 Processing form data - * @see HTML - * 5 - 4.10.18.3 Association of controls and forms - * @param source the source where the form is (to obtain further input elements) - * @param form the form - * @return the list - */ - private FormData prepareFormDataSet(Source source, Element form) { - List formDataFields = new LinkedList<>(); - - // Process each form field - Iterator it = getFormFields(source, form).iterator(); - while (it.hasNext()) { - FormField field = it.next(); - getLogger().debug("New form field: {}", field.getDebugInfo()); - for (String value : getDefaultTextValue(field)) { - formDataFields.add( - new FormDataField( - field.getName(), - value, - field.getFormControl().getFormControlType().isSubmit())); - } - } - return new FormData(formDataFields); - } - - private static FormFields getFormFields(Source source, Element form) { - SortedSet formControls = new TreeSet<>(); - addAll(formControls, form, HTMLElementName.INPUT); - addAll(formControls, form, HTMLElementName.TEXTAREA); - addAll(formControls, form, HTMLElementName.SELECT); - addAll(formControls, form, HTMLElementName.BUTTON); - - String formId = form.getAttributeValue("id"); - if (formId != null && !formId.isEmpty()) { - addAll(formControls, source.getAllElements("form", formId, true)); - } - for (Iterator it = formControls.iterator(); it.hasNext(); ) { - FormControl formControl = it.next(); - String targetForm = formControl.getAttributesMap().get("form"); - if (targetForm != null && !targetForm.equals(formId)) { - it.remove(); - } - } - - return new FormFields(formControls); - } - - private static void addAll( - SortedSet formControls, Segment segment, String tagName) { - addAll(formControls, segment.getAllElements(tagName)); - } - - private static void addAll(SortedSet formControls, List elements) { - for (Element element : elements) { - FormControl formControl = element.getFormControl(); - if (formControl != null) { - formControls.add(formControl); - } - } - } - - /** - * Gets the values for the given {@code field}. If the field is of submit type it passes the - * predefined values to the ValueGenerator and returns its predefined values. Gets the default - * value that the input field, including HTML5 types, should have. - * - * @param field the field - * @return a list with the values - */ - private List getDefaultTextValue(FormField field) { - - // Get the Id - String fieldId = field.getName(); - - // Create new HashMap 'fieldAttributes' and new list 'definedValues' - Map fieldAttributes = new HashMap<>(); - List definedValues = new ArrayList<>(); - - // Store all values in the FormFiled field into the Map 'fieldAttributes' - fieldAttributes.putAll(field.getFormControl().getAttributesMap()); - - // Places a key, Control Type, for each FormControlType - fieldAttributes.put("Control Type", field.getFormControl().getFormControlType().name()); - - // Handles Submit Fields - if (field.getFormControl().getFormControlType().isSubmit()) { - List submitFields = new ArrayList<>(); - for (String value : field.getPredefinedValues()) { - String finalValue = - this.valueGenerator.getValue( - uri, - url, - fieldId, - value, - definedValues, - envAttributes, - fieldAttributes); - submitFields.add(finalValue); - } - return submitFields; - } - - // Get its value(s) - List values = field.getValues(); - String defaultValue = null; - - // If the field has a value attribute present(Predefined value) - // Should store the value being submitted to be passed to the ValueGenerator - if (field.getFormControl().getAttributesMap().containsKey("value")) { - defaultValue = field.getFormControl().getAttributesMap().get("value"); - } - - getLogger().debug("Existing values: {}", values); - - // If there are no values at all or only an empty value - if (values.isEmpty() || (values.size() == 1 && values.get(0).isEmpty())) { - - // Check if we can use predefined values - Collection predefValues = field.getPredefinedValues(); - if (!predefValues.isEmpty()) { - // Store those predefined values in a list for the DefaultValueGenerator - definedValues.addAll(predefValues); - if (defaultValue == null) { - // Try first elements - Iterator iterator = predefValues.iterator(); - defaultValue = iterator.next(); - - // If there are more values, don't use the first, as it usually is a "No select" - // item - if (iterator.hasNext()) { - defaultValue = iterator.next(); - } - } - } - defaultValue = defaultValue == null ? DEFAULT_EMPTY_VALUE : defaultValue; - - } else if (defaultValue == null) { - defaultValue = values.get(0); - } - - // Get the default value used in DefaultValueGenerator - String finalValue = - this.valueGenerator.getValue( - uri, - url, - fieldId, - defaultValue, - definedValues, - envAttributes, - fieldAttributes); - - getLogger().debug("Generated: {}For field {}", finalValue, field.getName()); - - values = new ArrayList<>(1); - values.add(finalValue); - - return values; - } - - /** - * Notifies listeners that a new POST resource was found. - * - * @param message the source message - * @param depth the current depth - * @param url the URL of the resource - * @param requestBody the request body - * @see #notifyListenersPostResourceFound(HttpMessage, int, String, String) - */ - private void notifyPostResourceFound( - HttpMessage message, int depth, String url, String requestBody) { - getLogger() - .debug( - "Submitting form with POST method and message body with form parameters (normal encoding): {}", - requestBody); - notifyListenersResourceFound( - SpiderResourceFound.builder() - .setMessage(message) - .setDepth(depth + 1) - .setUri(url) - .setMethod(METHOD_POST) - .setBody(requestBody) - .build()); - } - - @Override - public boolean canParseResource(HttpMessage message, String path, boolean wasAlreadyConsumed) { - // Fallback parser - if it's a HTML message which has not already been processed - return !wasAlreadyConsumed && message.getResponseHeader().isHtml(); - } - - /** - * The fields (and its values) of a HTML form. - * - *

Builds the form data encoded with "application/x-www-form-urlencoded". - * - * @see HTML - * 4.01 Specification - 17.13.4 Form content types - */ - private static class FormData implements Iterable { - - private final List fields; - private final List submitFields; - - private FormData(List fields) { - this.fields = fields; - this.submitFields = new ArrayList<>(); - this.fields.forEach( - f -> { - if (f.isSubmit()) { - submitFields.add(f); - } - }); - } - - @Override - public Iterator iterator() { - return new IteratorImpl(); - } - - private class IteratorImpl implements Iterator { - - private boolean started; - private List consumedSubmitFields = new ArrayList<>(); - - @Override - public boolean hasNext() { - return !started || consumedSubmitFields.size() < submitFields.size(); - } - - @Override - public String next() { - if (!started) { - started = true; - } else if (consumedSubmitFields.size() >= submitFields.size()) { - throw new NoSuchElementException("No more form data to generate."); - } - - boolean submitted = false; - StringBuilder formData = new StringBuilder(100); - for (FormDataField field : fields) { - if (field.isSubmit()) { - if (submitted || consumedSubmitFields.contains(field)) { - continue; - } - submitted = true; - consumedSubmitFields.add(field); - } - - if (formData.length() > 0) { - formData.append('&'); - } - - formData.append(field.getName()); - formData.append('='); - formData.append(field.getValue()); - } - - return formData.toString(); - } - } - } - - /** A field of a {@link FormData}. */ - private static class FormDataField { - - private String name; - private String value; - private boolean submit; - - public FormDataField(String name, String value, boolean submit) { - try { - this.name = URLEncoder.encode(name, ENCODING_TYPE); - this.value = URLEncoder.encode(value, ENCODING_TYPE); - this.submit = submit; - } catch (UnsupportedEncodingException ignore) { - // UTF-8 is one of the standard charsets of the JVM. - } - } - - public String getName() { - return name; - } - - public String getValue() { - return value; - } - - public boolean isSubmit() { - return submit; - } - } -} diff --git a/zap/src/main/java/org/zaproxy/zap/spider/parser/SpiderHtmlParser.java b/zap/src/main/java/org/zaproxy/zap/spider/parser/SpiderHtmlParser.java deleted file mode 100644 index c86fa966eed..00000000000 --- a/zap/src/main/java/org/zaproxy/zap/spider/parser/SpiderHtmlParser.java +++ /dev/null @@ -1,433 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2012 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.spider.parser; - -import java.util.Arrays; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import net.htmlparser.jericho.Element; -import net.htmlparser.jericho.HTMLElementName; -import net.htmlparser.jericho.Source; -import net.htmlparser.jericho.StartTag; -import net.htmlparser.jericho.StartTagType; -import org.parosproxy.paros.network.HttpMessage; - -/** - * The Class SpiderHtmlParser is used for parsing of HTML files, gathering resource urls from them. - * - *

NOTE: Handling of HTML Forms is not done in this Parser. Instead see {@link - * SpiderHtmlFormParser}. - * - * @deprecated (2.12.0) See the spider add-on in zap-extensions instead. - */ -@Deprecated -public class SpiderHtmlParser extends SpiderParser { - - /** The Constant URL_PATTERN defining the pattern for a meta URL. */ - static final Pattern URL_PATTERN = - Pattern.compile( - "(?:url\\s*=|report-uri)\\s*[\"']?([^;'\"]+)", Pattern.CASE_INSENSITIVE); - - private static final Pattern PLAIN_COMMENTS_URL_PATTERN = - Pattern.compile( - "(?:http(?:s?):)?//[^\\x00-\\x1f\"'\\s<>#()\\[\\]{}]+", - Pattern.CASE_INSENSITIVE); - - private static final Pattern INLINE_CONTENT_URL_PATTERN = - Pattern.compile( - "(?:http(?:s?)://|(?:\\s|\\B)//?)[^\\x00-\\x1f\"'\\s<>#()\\[\\]{}]+", - Pattern.CASE_INSENSITIVE); - - private static final Pattern SRCSET_PATTERN = - Pattern.compile("[^\"'=\\s,]+\\.[^\\s,]+", Pattern.CASE_INSENSITIVE); - - private static final List elementsWithText = - Arrays.asList( - HTMLElementName.P, - HTMLElementName.TITLE, - HTMLElementName.H1, - HTMLElementName.H2, - HTMLElementName.H3, - HTMLElementName.H4, - HTMLElementName.H5, - HTMLElementName.H6, - HTMLElementName.LI, - HTMLElementName.BLOCKQUOTE); - - /** - * Functional interface that allows custom handling of URLs retrieved in attributes, to be - * manipulated before the URL is sent to a {@link SpiderParser#processURL} call. - */ - @FunctionalInterface - private interface CustomUrlProcessor { - void process(HttpMessage message, int depth, String localURL, String baseURL); - } - - private static final String IMPORT_TAG = "IMPORT"; - - private boolean baseTagSet; - - /** - * Instantiates a new spider html parser. - * - * @param params the params - * @throws NullPointerException if {@code params} is null. - */ - public SpiderHtmlParser(org.zaproxy.zap.spider.SpiderParam params) { - super(params); - } - - /** - * @throws NullPointerException if {@code message} is null. - */ - @Override - public boolean parseResource(HttpMessage message, Source source, int depth) { - - // Prepare the source, if not provided - if (source == null) { - source = new Source(message.getResponseBody().toString()); - } - - // Get the context (base URL) - String baseURL = message.getRequestHeader().getURI().toString(); - - // Try to see if there's any BASE tag that could change the base URL - Element base = source.getFirstElement(HTMLElementName.BASE); - if (base != null) { - getLogger().debug("Base tag was found in HTML: {}", base.getDebugInfo()); - String href = base.getAttributeValue("href"); - if (href != null && !href.isEmpty()) { - baseURL = getCanonicalURL(href, baseURL); - baseTagSet = true; - } - } - - // Parse the source - parseSource(message, source, depth, baseURL); - - // Parse the comments - if (getSpiderParam().isParseComments()) { - List comments = source.getAllStartTags(StartTagType.COMMENT); - for (StartTag comment : comments) { - Source s = new Source(comment.getTagContent()); - if (!parseSource(message, s, depth, baseURL)) { - Matcher matcher = PLAIN_COMMENTS_URL_PATTERN.matcher(s.toString()); - while (matcher.find()) { - processURL(message, depth, matcher.group(), baseURL); - } - } - } - } - - // Parse the DOCTYPEs (should only be one, but you never know;) - List doctypes = source.getAllStartTags(StartTagType.DOCTYPE_DECLARATION); - for (StartTag doctype : doctypes) { - for (String str : doctype.getTagContent().toString().split(" ")) { - if (str.startsWith("\"") && str.endsWith("\"")) { - processURL(message, depth, str.substring(1, str.length() - 1), baseURL); - } - } - } - - return false; - } - - /** - * Implements the CustomUrlProcessor to handle the special format of a srcset attribute element, - * see https://developer.mozilla.org/en-US/docs/Web/HTML/Element/img#attr-srcset the srcset - * attribute contains one or more strings separated by commas, indicating possible image sources - * for the user agent to use. - */ - private void srcSetProcessor(HttpMessage message, int depth, String localURL, String baseURL) { - Matcher results = SRCSET_PATTERN.matcher(localURL); - while (results.find()) { - if (!results.group().isEmpty()) { - processURL(message, depth, results.group(), baseURL); - } - } - } - - /** - * Parses the HTML Jericho source for the elements that contain references to other resources. - * - * @param message the message - * @param source the source - * @param depth the depth - * @param baseURL the base URL - * @return {@code true} if at least one URL was found, {@code false} otherwise. - */ - private boolean parseSource(HttpMessage message, Source source, int depth, String baseURL) { - getLogger().debug("Parsing an HTML message..."); - boolean resourcesfound = false; - // Process A elements - List elements = source.getAllElements(HTMLElementName.A); - for (Element el : elements) { - resourcesfound |= processAttributeElement(message, depth, baseURL, el, "href"); - resourcesfound |= processAttributeElement(message, depth, baseURL, el, "ping"); - } - - // Process Applet elements - elements = source.getAllElements(HTMLElementName.APPLET); - for (Element el : elements) { - resourcesfound |= processAttributeElement(message, depth, baseURL, el, "archive"); - resourcesfound |= processAttributeElement(message, depth, baseURL, el, "codebase"); - resourcesfound |= processAttributeElement(message, depth, baseURL, el, "src"); - } - - // Process AREA elements - elements = source.getAllElements(HTMLElementName.AREA); - for (Element el : elements) { - resourcesfound |= processAttributeElement(message, depth, baseURL, el, "href"); - resourcesfound |= processAttributeElement(message, depth, baseURL, el, "ping"); - } - - // Process AUDIO elements - elements = source.getAllElements(HTMLElementName.AUDIO); - for (Element el : elements) { - resourcesfound |= processAttributeElement(message, depth, baseURL, el, "src"); - } - - // Process Embed Elements - elements = source.getAllElements(HTMLElementName.EMBED); - for (Element el : elements) { - resourcesfound |= processAttributeElement(message, depth, baseURL, el, "src"); - } - - // Process Frame Elements - elements = source.getAllElements(HTMLElementName.FRAME); - for (Element el : elements) { - resourcesfound |= processAttributeElement(message, depth, baseURL, el, "src"); - } - - // Process IFrame Elements - elements = source.getAllElements(HTMLElementName.IFRAME); - for (Element el : elements) { - resourcesfound |= processAttributeElement(message, depth, baseURL, el, "src"); - } - - // Process Input elements - elements = source.getAllElements(HTMLElementName.INPUT); - for (Element el : elements) { - resourcesfound |= processAttributeElement(message, depth, baseURL, el, "src"); - } - - // Process ISINDEX elements - elements = source.getAllElements(HTMLElementName.ISINDEX); - for (Element el : elements) { - resourcesfound |= processAttributeElement(message, depth, baseURL, el, "action"); - } - - // Process Link elements - elements = source.getAllElements(HTMLElementName.LINK); - for (Element el : elements) { - resourcesfound |= processAttributeElement(message, depth, baseURL, el, "href"); - } - - // Process Object elements - elements = source.getAllElements(HTMLElementName.OBJECT); - for (Element el : elements) { - resourcesfound |= processAttributeElement(message, depth, baseURL, el, "data"); - resourcesfound |= processAttributeElement(message, depth, baseURL, el, "codebase"); - } - - // Process Script elements with src - elements = source.getAllElements(HTMLElementName.SCRIPT); - for (Element el : elements) { - resourcesfound |= processAttributeElement(message, depth, baseURL, el, "src"); - } - - // Process Table elements - elements = source.getAllElements(HTMLElementName.TABLE); - for (Element el : elements) { - resourcesfound |= processAttributeElement(message, depth, baseURL, el, "background"); - } - - // Process TD elements - elements = source.getAllElements(HTMLElementName.TD); - for (Element src : elements) { - resourcesfound |= processAttributeElement(message, depth, baseURL, src, "background"); - } - - // Process Video elements - elements = source.getAllElements(HTMLElementName.VIDEO); - for (Element el : elements) { - resourcesfound |= processAttributeElement(message, depth, baseURL, el, "src"); - List videoSourceElements = el.getAllElements(HTMLElementName.SOURCE); - for (Element sourceElement : videoSourceElements) { - resourcesfound |= - processAttributeElement(message, depth, baseURL, sourceElement, "src"); - } - resourcesfound |= processAttributeElement(message, depth, baseURL, el, "poster"); - } - - // Process Img elements - elements = source.getAllElements(HTMLElementName.IMG); - for (Element el : elements) { - resourcesfound |= processAttributeElement(message, depth, baseURL, el, "src"); - resourcesfound |= processAttributeElement(message, depth, baseURL, el, "longdesc"); - resourcesfound |= processAttributeElement(message, depth, baseURL, el, "lowsrc"); - resourcesfound |= processAttributeElement(message, depth, baseURL, el, "dynsrc"); - resourcesfound |= - processAttributeElement( - message, depth, baseURL, el, "srcset", this::srcSetProcessor); - } - - // Process IMPORT elements - elements = source.getAllElements(IMPORT_TAG); - for (Element el : elements) { - resourcesfound |= - processAttributeElement(message, depth, baseURL, el, "implementation"); - } - - // Process content of container tags which hold text - String baseUrlForText = baseURL; - for (String tag : elementsWithText) { - elements = source.getAllElements(tag); - for (Element el : elements) { - Matcher matcher = - INLINE_CONTENT_URL_PATTERN.matcher( - el.getContent().getRenderer().setMaxLineLength(0).toString()); - while (matcher.find()) { - String foundMatch = matcher.group().trim(); - if (baseTagSet) { - if (!baseUrlForText.endsWith("/")) { - baseUrlForText += "/"; - } - if (foundMatch.charAt(0) == '/' && foundMatch.indexOf("//") != 0) { - foundMatch = foundMatch.substring(1); - } - } - processURL(message, depth, foundMatch, baseUrlForText); - resourcesfound = true; - } - } - } - - // Process META elements - elements = source.getAllElements(HTMLElementName.META); - for (Element el : elements) { - // If we have http-equiv attribute, then urls can be found. - String equiv = el.getAttributeValue("http-equiv"); - String name = el.getAttributeValue("name"); - String content = el.getAttributeValue("content"); - if (equiv != null && content != null) { - - // For the following cases: - // http-equiv="refresh" content="0;URL=http://foo.bar/..." - // http-equiv="location" content="url=http://foo.bar/..." - if (equiv.equalsIgnoreCase("refresh") - || equiv.equalsIgnoreCase("location") - || equiv.equalsIgnoreCase("content-security-policy")) { - Matcher matcher = URL_PATTERN.matcher(content); - if (matcher.find()) { - String url = matcher.group(1); - processURL(message, depth, url, baseURL); - resourcesfound = true; - } - } - } else if ("msapplication-config".equalsIgnoreCase(name) - && content != null - && !content.equals("") - && !content.equalsIgnoreCase("none")) { - processURL(message, depth, content, baseURL); - resourcesfound = true; - } - } - - // Process HTML manifest elements - elements = source.getAllElements(HTMLElementName.HTML); - for (Element el : elements) { - resourcesfound |= processAttributeElement(message, depth, baseURL, el, "manifest"); - } - - // Process BODY background elements - elements = source.getAllElements(HTMLElementName.BODY); - for (Element el : elements) { - resourcesfound |= processAttributeElement(message, depth, baseURL, el, "background"); - } - - return resourcesfound; - } - - /** - * Processes the attribute with the given name of a Jericho element, for an URL. If an URL is - * found, notifies the listeners. - * - * @param message the message - * @param depth the depth - * @param baseURL the base URL - * @param element the element - * @param attributeName the attribute name - * @return {@code true} if a URL was processed, {@code false} otherwise. - */ - private boolean processAttributeElement( - HttpMessage message, int depth, String baseURL, Element element, String attributeName) { - return processAttributeElement(message, depth, baseURL, element, attributeName, null); - } - - /** - * Processes the attribute with the given name of a Jericho element, for an URL. If an URL is - * found, notifies the listeners. - * - * @param message the message - * @param depth the depth - * @param baseURL the base URL - * @param element the element - * @param attributeName the attribute name - * @param customUrlProcessor functional interface for custom manipulation of urls - * @return {@code true} if a URL was processed, {@code false} otherwise. - */ - private boolean processAttributeElement( - HttpMessage message, - int depth, - String baseURL, - Element element, - String attributeName, - CustomUrlProcessor customUrlProcessor) { - // The URL as written in the attribute (can be relative or absolute) - String localURL = element.getAttributeValue(attributeName); - if (localURL == null) { - return false; - } - - if (customUrlProcessor != null) { - customUrlProcessor.process(message, depth, localURL, baseURL); - } else if (!attributeName.equalsIgnoreCase("ping")) { - processURL(message, depth, localURL, baseURL); - } else { - for (String pingURL : localURL.split("\\s")) { - if (!pingURL.isEmpty()) { - processURL(message, depth, pingURL, baseURL); - } - } - } - return true; - } - - /** - * @throws NullPointerException if {@code message} is null. - */ - @Override - public boolean canParseResource(HttpMessage message, String path, boolean wasAlreadyConsumed) { - // Fallback parser - if it's a HTML message which has not already been processed - return !wasAlreadyConsumed && message.getResponseHeader().isHtml(); - } -} diff --git a/zap/src/main/java/org/zaproxy/zap/spider/parser/SpiderHttpHeaderParser.java b/zap/src/main/java/org/zaproxy/zap/spider/parser/SpiderHttpHeaderParser.java deleted file mode 100644 index c6d71d31730..00000000000 --- a/zap/src/main/java/org/zaproxy/zap/spider/parser/SpiderHttpHeaderParser.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2022 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.spider.parser; - -import java.util.regex.Matcher; -import net.htmlparser.jericho.Source; -import org.parosproxy.paros.network.HttpHeader; -import org.parosproxy.paros.network.HttpMessage; - -/** - * The Class SpiderHttpHeaderParser is used for parsing of HTTP headers that can include URLs. - * - * @see SpiderRedirectParser - * @deprecated (2.12.0) See the spider add-on in zap-extensions instead. - */ -@Deprecated -public class SpiderHttpHeaderParser extends SpiderParser { - - public SpiderHttpHeaderParser(org.zaproxy.zap.spider.SpiderParam params) { - super(params); - } - - @Override - public boolean parseResource(HttpMessage message, Source source, int depth) { - String baseURL = message.getRequestHeader().getURI().toString(); - - // Content-location header - String location = message.getResponseHeader().getHeader(HttpHeader.CONTENT_LOCATION); - if (location != null && !location.isEmpty()) { - processURL(message, depth, location, baseURL); - } - // Refresh header - String refresh = message.getResponseHeader().getHeader(HttpHeader.REFRESH); - if (refresh != null && !refresh.isEmpty()) { - Matcher matcher = SpiderHtmlParser.URL_PATTERN.matcher(refresh); - if (matcher.find()) { - String url = matcher.group(1); - processURL(message, depth, url, baseURL); - } - } - - // Link header - potentially multiple absolute or relative URLs in < > - String link = message.getResponseHeader().getHeader(HttpHeader.LINK); - if (link != null && !link.isEmpty()) { - int offset = 0; - while (true) { - int i = link.indexOf("<", offset); - if (i < 0) { - break; - } - int j = link.indexOf(">", i); - if (j < 0) { - break; - } - processURL(message, depth, link.substring(i + 1, j), baseURL); - offset = j; - } - } - // We do not consider the message fully parsed - return false; - } - - @Override - public boolean canParseResource(HttpMessage message, String path, boolean wasAlreadyParsed) { - return true; - } -} diff --git a/zap/src/main/java/org/zaproxy/zap/spider/parser/SpiderODataAtomParser.java b/zap/src/main/java/org/zaproxy/zap/spider/parser/SpiderODataAtomParser.java deleted file mode 100644 index 7f5ca4755b8..00000000000 --- a/zap/src/main/java/org/zaproxy/zap/spider/parser/SpiderODataAtomParser.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2013 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.spider.parser; - -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import net.htmlparser.jericho.Source; -import org.apache.commons.lang3.StringEscapeUtils; -import org.parosproxy.paros.network.HttpMessage; - -/** - * Used to parse OData content in Atom format. - * - *

It's derived from the SpiderTextParser. Even if the format of the file is XML we will process - * it as a simple text file - * - * @deprecated (2.12.0) See the spider add-on in zap-extensions instead. - */ -@Deprecated -public class SpiderODataAtomParser extends SpiderParser { - - /** The Constant urlPattern defining the pattern for an url. */ - private static final Pattern patternURL = Pattern.compile("href=\\\"([\\w();&'/,=\\-]*)\\\""); - - /** the Constant patternBase defines the pattern for a base url */ - private static final Pattern patternBase = - Pattern.compile("base=\"(http(s?)://[^\\x00-\\x1f\"'\\s<>#]+)\""); - - public SpiderODataAtomParser() { - this(null); - } - - public SpiderODataAtomParser(org.zaproxy.zap.spider.SpiderParam param) { - super(param); - } - - @Override - public boolean parseResource(HttpMessage message, Source source, int depth) { - getLogger().debug("Parsing an OData Atom resource."); - - // Get the context (base url) - String baseURL = message.getRequestHeader().getURI().toString(); - - // Use a simple pattern matcher to find urls (absolute and relative) - - String bodyAsStr = message.getResponseBody().toString(); - - // Handle base tag if any - // xml:base="http://myserver:8001/remoting/myapp.svc/" - - Matcher matcher = patternBase.matcher(bodyAsStr); - if (matcher.find()) { - baseURL = matcher.group(1); - baseURL = StringEscapeUtils.unescapeXml(baseURL); - } - - boolean foundAtLeastOneResult = false; - matcher = patternURL.matcher(bodyAsStr); - while (matcher.find()) { - String s = matcher.group(1); - s = StringEscapeUtils.unescapeXml(s); - - processURL(message, depth, s, baseURL); - foundAtLeastOneResult = true; - } - - // resource is consumed only if at least one link is found - return foundAtLeastOneResult; - } - - @Override - public boolean canParseResource(HttpMessage message, String path, boolean wasAlreadyParsed) { - // Fallback parser - if it's an XML message which has not already been processed - return !wasAlreadyParsed && message.getResponseHeader().isXml(); - } -} diff --git a/zap/src/main/java/org/zaproxy/zap/spider/parser/SpiderParser.java b/zap/src/main/java/org/zaproxy/zap/spider/parser/SpiderParser.java deleted file mode 100644 index f7d33cd2b5b..00000000000 --- a/zap/src/main/java/org/zaproxy/zap/spider/parser/SpiderParser.java +++ /dev/null @@ -1,220 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2012 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.spider.parser; - -import java.util.LinkedList; -import java.util.List; -import java.util.Objects; -import net.htmlparser.jericho.Source; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.parosproxy.paros.network.HttpMessage; -import org.parosproxy.paros.network.HttpRequestHeader; - -/** - * The Abstract Class SpiderParser is the base for parsers used by the spider. The main purpose of - * these Parsers is to find links (uris) to resources in the provided content. Uses the Jericho - * Library for parsing. - * - * @deprecated (2.12.0) See the spider add-on in zap-extensions instead. - */ -@Deprecated -public abstract class SpiderParser { - - /** The listeners to spider parsing events. */ - private List listeners = new LinkedList<>(); - - /** - * The Constant log. - * - * @deprecated (2.10.0) Use {@link #getLogger()} instead. - */ - @Deprecated - protected static final org.apache.log4j.Logger log = - org.apache.log4j.Logger.getLogger(SpiderParser.class); - - private final Logger logger = LogManager.getLogger(getClass()); - - private org.zaproxy.zap.spider.SpiderParam spiderParam; - - public SpiderParser() {} - - public SpiderParser(org.zaproxy.zap.spider.SpiderParam spiderParam) { - this.spiderParam = - Objects.requireNonNull(spiderParam, "Parameter spiderParam must not be null."); - } - - public void setSpiderParam(org.zaproxy.zap.spider.SpiderParam spiderParam) { - this.spiderParam = spiderParam; - } - - protected org.zaproxy.zap.spider.SpiderParam getSpiderParam() { - return spiderParam; - } - - /** - * Gets the logger. - * - * @return the logger, never {@code null}. - * @since 2.10.0 - */ - protected Logger getLogger() { - return logger; - } - - /** - * Adds a listener to spider parsing events. - * - * @param listener the listener - */ - public void addSpiderParserListener(SpiderParserListener listener) { - listeners.add(listener); - } - - /** - * Removes a listener to spider parsing events. - * - * @param listener the listener - */ - public void removeSpiderParserListener(SpiderParserListener listener) { - this.listeners.remove(listener); - } - - /** - * Notify the listeners that a resource was found. - * - * @param resourceFound the resource found. - * @since 2.11.0 - */ - protected void notifyListenersResourceFound(SpiderResourceFound resourceFound) { - for (SpiderParserListener l : listeners) { - l.resourceFound(resourceFound); - } - } - - /** - * Notify the listeners that a resource was found. - * - * @param message the http message containing the response. - * @param depth the depth of this resource in the crawling tree - * @param uri the uri - * @deprecated (2.11.0) Use {@link #notifyListenersResourceFound(SpiderResourceFound)} instead. - */ - @Deprecated - protected void notifyListenersResourceFound(HttpMessage message, int depth, String uri) { - notifyListenersResourceFound( - SpiderResourceFound.builder() - .setMessage(message) - .setDepth(depth) - .setUri(uri) - .build()); - } - - /** - * Notify the listeners that a POST resource was found. You can read more about this call in the - * documentation for resourcePostURIFound in {@link SpiderParserListener}. - * - * @param message the http message containing the response. - * @param depth the depth of this resource in the crawling tree - * @param uri the uri - * @param requestBody the request body - * @deprecated (2.11.0) Use {@link #notifyListenersResourceFound(SpiderResourceFound)} instead. - */ - @Deprecated - protected void notifyListenersPostResourceFound( - HttpMessage message, int depth, String uri, String requestBody) { - notifyListenersResourceFound( - SpiderResourceFound.builder() - .setMessage(message) - .setDepth(depth) - .setUri(uri) - .setMethod(HttpRequestHeader.POST) - .setBody(requestBody) - .build()); - } - - /** - * Builds an url and notifies the listeners. - * - * @param message the message - * @param depth the depth - * @param localURL the local url - * @param baseURL the base url - */ - protected void processURL(HttpMessage message, int depth, String localURL, String baseURL) { - // Build the absolute canonical URL - String fullURL = getCanonicalURL(localURL, baseURL); - if (fullURL == null) { - return; - } - - getLogger().debug("Canonical URL constructed using '{}': {}", localURL, fullURL); - notifyListenersResourceFound( - SpiderResourceFound.builder() - .setMessage(message) - .setDepth(depth + 1) - .setUri(fullURL) - .build()); - } - - protected String getCanonicalURL(String localURL, String baseURL) { - return org.zaproxy.zap.spider.URLCanonicalizer.getCanonicalURL( - localURL, baseURL, spiderParam::isIrrelevantUrlParameter); - } - - /** - * Parses the resource. The HTTP message containing the request and the response is given. Also, - * if possible, a Jericho source with the Response Body is provided. - * - *

When a link is encountered, implementations can use {@link #processURL(HttpMessage, int, - * String, String)} and {@link #notifyListenersResourceFound(SpiderResourceFound)} to announce - * the found URIs. - * - *

The return value specifies whether the resource should be considered 'completely - * processed'/consumed and should be treated accordingly by subsequent parsers. For example, any - * parsers which are meant to be 'fall-back' parsers should skip messages already processed by - * other parsers. - * - * @param message the full http message containing the request and the response - * @param source a Jericho source with the Response Body from the HTTP message. This parameter - * can be {@code null}, in which case the parser implementation should ignore it. - * @param depth the depth of this resource - * @return whether the resource is considered to be exhaustively processed - */ - public abstract boolean parseResource(final HttpMessage message, Source source, int depth); - - /** - * Checks whether the parser should be called to parse the given HttpMessage. - * - *

Based on the specifics of the HttpMessage and whether this message was already processed - * by another Parser, this method should decide whether the {@link #parseResource(HttpMessage, - * Source, int)} should be invoked. - * - *

The {@code wasAlreadyConsumed} could be used by parsers which represent a 'fall-back' - * parser to check whether any other parser has processed the message before. - * - * @param message the full http message containing the request and the response - * @param path the resource path, provided for convenience - * @param wasAlreadyConsumed if the resource was already parsed by another SpiderParser - * @return true, if the {@link #parseResource(HttpMessage, Source, int)} should be invoked. - */ - public abstract boolean canParseResource( - final HttpMessage message, String path, boolean wasAlreadyConsumed); -} diff --git a/zap/src/main/java/org/zaproxy/zap/spider/parser/SpiderParserListener.java b/zap/src/main/java/org/zaproxy/zap/spider/parser/SpiderParserListener.java deleted file mode 100644 index e472bd631ad..00000000000 --- a/zap/src/main/java/org/zaproxy/zap/spider/parser/SpiderParserListener.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2012 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.spider.parser; - -/** - * The listener interface for receiving spiderParser events. The class that is interested in - * processing a spiderParser event implements this interface, and the object created with that class - * is registered with a component using the component's {@code addSpiderParserListener} method. When - * the spiderParser event occurs, that object's appropriate method is invoked. - * - * @deprecated (2.12.0) See the spider add-on in zap-extensions instead. - */ -@Deprecated -public interface SpiderParserListener { - - /** - * Event triggered when a new resource is found. The resourceFound contains all the required - * information about the resource (source message, URI, depth, method, etc.). - * - * @param resourceFound definition of found spider resource - */ - void resourceFound(SpiderResourceFound resourceFound); -} diff --git a/zap/src/main/java/org/zaproxy/zap/spider/parser/SpiderRedirectParser.java b/zap/src/main/java/org/zaproxy/zap/spider/parser/SpiderRedirectParser.java deleted file mode 100644 index d74dfc12700..00000000000 --- a/zap/src/main/java/org/zaproxy/zap/spider/parser/SpiderRedirectParser.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2013 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.spider.parser; - -import net.htmlparser.jericho.Source; -import org.parosproxy.paros.network.HttpHeader; -import org.parosproxy.paros.network.HttpMessage; -import org.parosproxy.paros.network.HttpStatusCode; - -/** - * The Class SpiderRedirectParser is used for parsing of HTTP Redirection messages. - * - * @deprecated (2.12.0) See the spider add-on in zap-extensions instead. - */ -@Deprecated -public class SpiderRedirectParser extends SpiderParser { - - public SpiderRedirectParser() { - this(null); - } - - public SpiderRedirectParser(org.zaproxy.zap.spider.SpiderParam param) { - super(param); - } - - @Override - public boolean parseResource(HttpMessage message, Source source, int depth) { - getLogger().debug("Parsing an HTTP redirection resource..."); - - String location = message.getResponseHeader().getHeader(HttpHeader.LOCATION); - if (location != null && !location.isEmpty()) { - // Include the base url as well as some applications send relative URLs instead of - // absolute ones - String baseURL = message.getRequestHeader().getURI().toString(); - processURL(message, depth, location, baseURL); - } - // We consider the message fully parsed - return true; - } - - @Override - public boolean canParseResource(HttpMessage message, String path, boolean wasAlreadyParsed) { - return HttpStatusCode.isRedirection(message.getResponseHeader().getStatusCode()); - } -} diff --git a/zap/src/main/java/org/zaproxy/zap/spider/parser/SpiderResourceFound.java b/zap/src/main/java/org/zaproxy/zap/spider/parser/SpiderResourceFound.java deleted file mode 100644 index 1846984adc6..00000000000 --- a/zap/src/main/java/org/zaproxy/zap/spider/parser/SpiderResourceFound.java +++ /dev/null @@ -1,371 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2021 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.spider.parser; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Objects; -import org.parosproxy.paros.network.HttpHeaderField; -import org.parosproxy.paros.network.HttpMessage; -import org.parosproxy.paros.network.HttpRequestHeader; - -/** - * Class SpiderResourceFound is used to store information about found resources by spider parsers. - * - * @since 2.11.0 - * @deprecated (2.12.0) See the spider add-on in zap-extensions instead. - */ -@Deprecated -public class SpiderResourceFound { - /** The message where the resource was found. */ - private final HttpMessage message; - - /** Spider depth for resource. */ - private final int depth; - - /** HTTP method for resource. */ - private final String method; - - /** Uniform resource identifier of resource. */ - private final String uri; - - /** Body for the resource. */ - private final String body; - - /** Defines resource as useful or not useful in the fetching process. */ - private final boolean shouldIgnore; - - /** Additional request headers to be passed for the resource. */ - private final List headers; - - /** - * Instantiates a spider resource found. - * - * @param message the message for the found resource - * @param depth the depth of this resource in the crawling process - * @param uri the universal resource locator - * @param method HTTP method for the resource found - * @param body request body for the resource - * @param shouldIgnore flag to ignore the resource found - * @param headers additional request headers for the resource - */ - private SpiderResourceFound( - HttpMessage message, - int depth, - String uri, - String method, - String body, - boolean shouldIgnore, - List headers) { - this.message = message; - this.depth = depth; - this.uri = uri; - this.method = method; - this.body = body; - this.shouldIgnore = shouldIgnore; - this.headers = headers; - } - - /** - * Returns the message where the resource was found. - * - * @return HTTP message - */ - public HttpMessage getMessage() { - return message; - } - - /** - * Returns the spider depth of the resource. - * - * @return depth value - */ - public int getDepth() { - return depth; - } - - /** - * Gives back the method to be applied for the found resource. - * - * @return HTTP method - */ - public String getMethod() { - return method; - } - - /** - * Returns the URI of the found resource. - * - * @return uniform resource identifier - */ - public String getUri() { - return uri; - } - - /** - * Returns request body for the found resource. - * - * @return body string (empty if resource is GET-based) - */ - public String getBody() { - return body; - } - - /** - * States if the found resource should be ignored in the fetching process. - * - * @return boolean whether resource should be ignored - */ - public boolean isShouldIgnore() { - return shouldIgnore; - } - - /** - * Returns the additional request headers for the resource found. - * - * @return list of HTTP header fields - */ - public List getHeaders() { - return headers; - } - - @Override - public int hashCode() { - return Objects.hash(body, depth, headers, message, method, shouldIgnore, uri); - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (!(obj instanceof SpiderResourceFound)) { - return false; - } - SpiderResourceFound other = (SpiderResourceFound) obj; - return Objects.equals(body, other.body) - && depth == other.depth - && Objects.equals(headers, other.headers) - && Objects.equals(message, other.message) - && Objects.equals(method, other.method) - && shouldIgnore == other.shouldIgnore - && Objects.equals(uri, other.uri); - } - - @Override - public String toString() { - StringBuilder strBuilder = new StringBuilder(75); - strBuilder - .append("[Method=") - .append(method) - .append(", URI=") - .append(uri) - .append(", Headers=") - .append(headers) - .append(", Body=") - .append(body) - .append(", ShouldIgnore=") - .append(shouldIgnore) - .append(", Depth=") - .append(depth) - .append(", Message=") - .append(message); - return strBuilder.toString(); - } - - /** - * Gets a new builder for a spider resource found. - * - * @return a new spider resource found builder. - */ - public static Builder builder() { - return new Builder(); - } - - /** - * Gets a new spider resource found builder, using the values of the supplied resource found. - * - * @param spiderResourceFound spider resource found for reference. - * @return a new spider resource found builder. - * @throws IllegalArgumentException if the given parameter is {@code null}. - */ - public static Builder builder(SpiderResourceFound spiderResourceFound) { - if (spiderResourceFound == null) { - throw new IllegalArgumentException("Parameter spiderResourceFound must not be null."); - } - return new Builder(spiderResourceFound); - } - - /** Builder of {@link SpiderResourceFound}. */ - public static class Builder { - private HttpMessage message; - private int depth; - private String method; - private String uri; - private String body; - private boolean shouldIgnore; - private List headers; - - private Builder() { - this.message = null; - this.depth = 0; - this.method = HttpRequestHeader.GET; - this.uri = ""; - this.body = ""; - this.shouldIgnore = false; - this.headers = Collections.emptyList(); - } - - private Builder(SpiderResourceFound resourceFound) { - this.message = resourceFound.getMessage(); - this.depth = resourceFound.getDepth(); - this.method = resourceFound.getMethod(); - this.uri = resourceFound.getUri(); - this.body = resourceFound.getBody(); - this.shouldIgnore = resourceFound.isShouldIgnore(); - this.headers = resourceFound.getHeaders(); - } - - /** - * Sets the message where the resource was found. - * - * @param message HTTP message where the resource as found (null allowed). - * @return the builder. - */ - public Builder setMessage(HttpMessage message) { - this.message = message; - return this; - } - - /** - * Sets the spider depth for the resource found. - * - * @param depth Spider depth for resource. - * @return the builder. - * @throws IllegalArgumentException if the given parameter is negative. - */ - public Builder setDepth(int depth) { - if (depth < 0) { - throw new IllegalArgumentException("Parameter depth must not be negative."); - } - this.depth = depth; - return this; - } - - /** - * Sets the method for the resource found. - * - * @param method Method for resource found. - * @return the builder. - * @throws IllegalArgumentException if the given parameter is {@code null}. - */ - public Builder setMethod(String method) { - if (method == null) { - throw new IllegalArgumentException("Parameter method must not be null."); - } - this.method = method; - return this; - } - - /** - * Sets the URI for the resource found. - * - * @param uri URI for resource found. - * @return the builder. - * @throws IllegalArgumentException if the given parameter is {@code null}. - */ - public Builder setUri(String uri) { - if (uri == null) { - throw new IllegalArgumentException("Parameter uri must not be null."); - } - this.uri = uri; - return this; - } - - /** - * Sets the request body for the resource found. - * - * @param body Body for resource found. - * @return the builder. - * @throws IllegalArgumentException if the given parameter is {@code null}. - */ - public Builder setBody(String body) { - if (body == null) { - throw new IllegalArgumentException("Parameter body must not be null."); - } - this.body = body; - return this; - } - - /** - * Sets a flag whether the resource should be ignored. - * - * @param shouldIgnore Flag for resource found to be ignored. - * @return the builder. - */ - public Builder setShouldIgnore(boolean shouldIgnore) { - this.shouldIgnore = shouldIgnore; - return this; - } - - /** - * Sets additional request headers for the resource found. Only valid header fields (i.e. - * name not {@code null} and not empty, value not {@code null}) will be added. - * - * @param headers Additional request headers for resource found. - * @return the builder. - * @throws IllegalArgumentException if the given parameter is {@code null} or an element is - * {@code null}. - */ - public Builder setHeaders(List headers) { - if (headers == null) { - throw new IllegalArgumentException("Parameter headers must not be null."); - } - if (!headers.isEmpty()) { - List validHeaders = new ArrayList<>(); - for (HttpHeaderField headerField : headers) { - if (headerField == null) { - throw new IllegalArgumentException("Element of headers must not be null."); - } - if (headerField.getName() != null - && !headerField.getName().trim().isEmpty() - && headerField.getValue() != null) { - validHeaders.add(headerField); - } - } - this.headers = Collections.unmodifiableList(validHeaders); - } else { - this.headers = Collections.emptyList(); - } - return this; - } - - /** - * Builds a new {@code SpiderResourceFound}, with the configurations previously set. - * - * @return a new {@code SpiderResourceFound}. - */ - public SpiderResourceFound build() { - return new SpiderResourceFound( - message, depth, uri, method, body, shouldIgnore, headers); - } - } -} diff --git a/zap/src/main/java/org/zaproxy/zap/spider/parser/SpiderRobotstxtParser.java b/zap/src/main/java/org/zaproxy/zap/spider/parser/SpiderRobotstxtParser.java deleted file mode 100644 index 0860a6bb20b..00000000000 --- a/zap/src/main/java/org/zaproxy/zap/spider/parser/SpiderRobotstxtParser.java +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2012 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.spider.parser; - -import java.util.StringTokenizer; -import net.htmlparser.jericho.Source; -import org.parosproxy.paros.network.HttpMessage; - -/** - * The Class SpiderRobotstxtParser used for parsing Robots.txt files. - * - * @since 2.0.0 - * @deprecated (2.12.0) See the spider add-on in zap-extensions instead. - */ -@Deprecated -public class SpiderRobotstxtParser extends SpiderParser { - - private static final String COMMENT_TOKEN = "#"; - - private static final String PATTERNS_DISALLOW = "(?i)Disallow:.*"; - private static final String PATTERNS_ALLOW = "(?i)Allow:.*"; - - private static final int PATTERNS_DISALLOW_LENGTH = 9; - private static final int PATTERNS_ALLOW_LENGTH = 6; - - /** - * Instantiates a new spider robotstxt parser. - * - * @param params the params - * @throws NullPointerException if {@code params} is null. - */ - public SpiderRobotstxtParser(org.zaproxy.zap.spider.SpiderParam params) { - super(params); - } - - /** - * @throws NullPointerException if {@code message} is null. - */ - @Override - public boolean parseResource(HttpMessage message, Source source, int depth) { - if (!getSpiderParam().isParseRobotsTxt()) { - return false; - } - getLogger().debug("Parsing a robots.txt resource..."); - - String baseURL = message.getRequestHeader().getURI().toString(); - - StringTokenizer st = new StringTokenizer(message.getResponseBody().toString(), "\n"); - while (st.hasMoreTokens()) { - String line = st.nextToken(); - - int commentStart = line.indexOf(COMMENT_TOKEN); - if (commentStart != -1) { - line = line.substring(0, commentStart); - } - - // remove HTML markup and clean - line = line.replaceAll("<[^>]+>", ""); - line = line.trim(); - - if (line.isEmpty()) { - continue; - } - getLogger().debug("Processing robots.txt line: {}", line); - - if (line.matches(PATTERNS_DISALLOW)) { - processPath(message, depth, line.substring(PATTERNS_DISALLOW_LENGTH), baseURL); - } else if (line.matches(PATTERNS_ALLOW)) { - processPath(message, depth, line.substring(PATTERNS_ALLOW_LENGTH), baseURL); - } - } - - // We consider the message fully parsed, so it doesn't get parsed by 'fallback' parsers - return true; - } - - private void processPath(HttpMessage message, int depth, String path, String baseURL) { - String processedPath = path.trim(); - if (processedPath.endsWith("*")) { - processedPath = processedPath.substring(0, processedPath.length() - 1).trim(); - } - - if (!processedPath.isEmpty()) { - processURL(message, depth, processedPath, baseURL); - } - } - - @Override - public boolean canParseResource(HttpMessage message, String path, boolean wasAlreadyParsed) { - // If it's a robots.txt file - return "/robots.txt".equalsIgnoreCase(path); - } -} diff --git a/zap/src/main/java/org/zaproxy/zap/spider/parser/SpiderSVNEntriesParser.java b/zap/src/main/java/org/zaproxy/zap/spider/parser/SpiderSVNEntriesParser.java deleted file mode 100644 index 78497cc14c1..00000000000 --- a/zap/src/main/java/org/zaproxy/zap/spider/parser/SpiderSVNEntriesParser.java +++ /dev/null @@ -1,446 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2014 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.spider.parser; - -import java.io.ByteArrayInputStream; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.ParserConfigurationException; -import net.htmlparser.jericho.Source; -import org.apache.logging.log4j.LogManager; -import org.parosproxy.paros.network.HttpMessage; -import org.w3c.dom.Document; -import org.w3c.dom.Element; -import org.w3c.dom.Node; -import org.w3c.dom.NodeList; -import org.xml.sax.InputSource; -import org.xml.sax.SAXException; -import org.zaproxy.zap.utils.XmlUtils; - -/** - * The Class SpiderSVNEntriesParser is used for parsing SVN metadata, including SVN "entries" and - * "wc.db" files. - * - * @author 70pointer - * @deprecated (2.12.0) See the spider add-on in zap-extensions instead. - */ -@Deprecated -public class SpiderSVNEntriesParser extends SpiderParser { - /* this class was Cloned from SpiderRobotstxtParser, by Cosmin. Credit where credit is due. */ - - /** a pattern to match for SQLite based file (in ".svn/wc.db") */ - private static final Pattern svnSQLiteFormatPattern = Pattern.compile("^SQLite format "); - - /** a pattern to match for XML based entries files */ - private static final Pattern svnXMLFormatPattern = Pattern.compile("= SVN working copy format 12, or >= SVN 1.7) - File tempSqliteFile; - try { - // get the binary data, and put it in a temp file we can use with the SQLite JDBC - // driver - // Note: File is not AutoCloseable, so cannot use a "try with resources" to manage - // it - tempSqliteFile = File.createTempFile("sqlite", null); - tempSqliteFile.deleteOnExit(); - OutputStream fos = new FileOutputStream(tempSqliteFile); - fos.write(message.getResponseBody().getBytes()); - fos.close(); - - // now load the temporary SQLite file using JDBC, and query the file entries within. - Class.forName("org.sqlite.JDBC"); - String sqliteConnectionUrl = "jdbc:sqlite:" + tempSqliteFile.getAbsolutePath(); - - try (Connection conn = DriverManager.getConnection(sqliteConnectionUrl)) { - if (conn != null) { - Statement stmt = null; - ResultSet rsSVNWCFormat = null; - ResultSet rsNodes = null; - ResultSet rsRepo = null; - try { - stmt = conn.createStatement(); - rsSVNWCFormat = stmt.executeQuery("pragma USER_VERSION"); - - // get the precise internal version of SVN in use - // this will inform how the Spider recurse should proceed in an - // efficient manner. - int svnFormat = 0; - while (rsSVNWCFormat.next()) { - getLogger().debug("Got a row from 'pragma USER_VERSION'"); - svnFormat = rsSVNWCFormat.getInt(1); - break; - } - if (svnFormat < 29) { - throw new Exception( - "The SVN Working Copy Format of the SQLite database should be >= 29. We found " - + svnFormat); - } - if (svnFormat > 31) { - throw new Exception( - "SVN Working Copy Format " - + svnFormat - + " is not supported at this time. We support up to and including format 31 (~ SVN 1.8.5)"); - } - getLogger() - .debug( - "Internal SVN Working Copy Format for {} is {}", - tempSqliteFile, - svnFormat); - getLogger() - .debug( - "Refer to http://svn.apache.org/repos/asf/subversion/trunk/subversion/libsvn_wc/wc.h for more details!"); - - // allow future changes to be easily handled - switch (svnFormat) { - case 29: - case 30: - case 31: - rsNodes = - stmt.executeQuery( - "select kind,local_relpath,'pristine/'||substr(checksum,7,2) || \"/\" || substr(checksum,7)|| \".svn-base\" from nodes order by wc_id"); - break; - } - - if (rsNodes == null) { - getLogger() - .error( - "There was a problem parsing the resource. rsNodes should not be null."); - return false; - } - // now get the list of files stored in the SVN repo (or this folder of - // the repo, depending the SVN working copy format in use) - while (rsNodes.next()) { - getLogger() - .debug( - "Got a Node from the SVN wc.db file (format {})", - svnFormat); - String kind = rsNodes.getString(1); - String filename = rsNodes.getString(2); - String svn_filename = rsNodes.getString(3); - - if (filename != null && filename.length() > 0) { - getLogger() - .debug( - "Found a file/directory name in the (SQLite based) SVN wc.db file"); - - processURL( - message, - depth, - "../" + filename + (kind.equals("dir") ? "/" : ""), - baseURL); - - // re-seed the spider for this directory. - // this is not to do with the SVN version, but in case the SVN - // root is not the WEB root.. - // in order to be sure we catch all the SVN repos, we recurse. - if (kind.equals("dir")) { - processURL( - message, - depth, - "../" + filename + "/.svn/wc.db", - baseURL); - } - // if we have an internal SVN filename for the file, process it. - // this will probably result in source code disclosure at some - // point. - if (kind.equals("file") - && svn_filename != null - && svn_filename.length() > 0) { - processURL(message, depth, svn_filename, baseURL); - } - } - } - - rsRepo = stmt.executeQuery("select root from REPOSITORY order by id"); - // get additional information on where the SVN repository is located - while (rsRepo.next()) { - getLogger() - .debug( - "Got a potential Repository from the SVN wc.db file (format {})", - svnFormat); - String repos_path = rsRepo.getString(1); - if (repos_path != null && repos_path.length() > 0) { - // exclude local repositories here.. we cannot retrieve or - // spider them - Matcher repoMatcher = - svnRepoLocationPattern.matcher(repos_path); - if (repoMatcher.find()) { - getLogger() - .debug( - "Found an SVN repository location in the (SQLite based) SVN wc.db file"); - processURL(message, depth, repos_path + "/", baseURL); - } - } - } - } catch (Exception e) { - getLogger() - .error( - "Error executing SQL on temporary SVN SQLite database '{}': ", - sqliteConnectionUrl, - e); - } finally { - // the JDBC driver in use does not play well with "try with resource" - // construct. I tried! - if (rsRepo != null) rsRepo.close(); - if (rsNodes != null) rsNodes.close(); - if (rsSVNWCFormat != null) rsSVNWCFormat.close(); - if (stmt != null) stmt.close(); - } - } else - throw new SQLException( - "Could not open a JDBC connection to SQLite file " - + tempSqliteFile.getAbsolutePath()); - } catch (Exception e) { - // the connection will have been closed already, since we're used a try with - // resources - getLogger() - .error( - "Error parsing temporary SVN SQLite database {}", - sqliteConnectionUrl); - } finally { - // delete the temp file. - // this will be deleted when the VM is shut down anyway, but better to be safe - // than to run out of disk space. - tempSqliteFile.delete(); - } - - } catch (IOException | ClassNotFoundException e) { - getLogger() - .error( - "An error occurred trying to set up to parse the SQLite based file: ", - e); - // We consider the message fully parsed, so it doesn't get parsed by 'fallback' - // parsers - return true; - } - - } else if (svnXMLFormatMatcher.find()) { - // XML format is being used, ( < SVN working copy format 7). - // The XML based file was replaced with the text based format with SVN 1.4, when format - // 8 went live - // Not all the working copy formats went live in SVN versions, so tracking the format - // against the SVN version is tricky. - - Document doc; - try { - // work around the "no protocol" issue by wrapping the content in a - // ByteArrayInputStream - doc = - dBuilder.parse( - new InputSource( - new ByteArrayInputStream(content.getBytes("utf-8")))); - } catch (SAXException | IOException e) { - getLogger() - .error( - "An error occurred trying to parse the XML based .svn/entries file: ", - e); - // We consider the message fully parsed, so it doesn't get parsed by 'fallback' - // parsers - return true; - } - NodeList nodelist = doc.getElementsByTagName("entry"); - for (int i = 0; i < nodelist.getLength(); i++) { - Node svnEntryNode = nodelist.item(i); - String svnEntryName = ((Element) svnEntryNode).getAttribute("name"); - String svnEntryKind = ((Element) svnEntryNode).getAttribute("kind"); - String svnEntryUrl = ((Element) svnEntryNode).getAttribute("url"); - String svnEntryCopyFromUrl = ((Element) svnEntryNode).getAttribute("copyfrom-url"); - - if (svnEntryName != null && svnEntryName.length() > 0) { - getLogger() - .debug( - "Found a file/directory name in the (XML based) SVN < 1.4 entries file"); - processURL( - message, - depth, - "../" + svnEntryName + (svnEntryKind.equals("dir") ? "/" : ""), - baseURL); - // get the internal SVN file, probably leading to source code disclosure - if (svnEntryKind.equals("file")) { - processURL( - message, depth, "text-base/" + svnEntryName + ".svn-base", baseURL); - } - // re-seed the spider for this directory. - if (svnEntryKind.equals("dir")) { - processURL(message, depth, "../" + svnEntryName + "/.svn/entries", baseURL); - } - } - - // expected to be true for the first entry only (the directory housing other - // entries) - if (svnEntryName != null - && svnEntryName.length() == 0 - && svnEntryKind.equals("dir")) { - // exclude local repositories here.. we cannot retrieve or spider them - Matcher repoMatcher = svnRepoLocationPattern.matcher(svnEntryUrl); - if (repoMatcher.find()) { - getLogger() - .debug( - "Found an SVN repository location in the (XML based) SVN < 1.4 entries file"); - processURL(message, depth, svnEntryUrl + "/", baseURL); - } - } - // this attribute seems to be set on various entries. Correspond to files, rather - // than directories - Matcher urlMatcher = svnRepoLocationPattern.matcher(svnEntryCopyFromUrl); - if (urlMatcher.find()) { - getLogger().debug("Found an SVN URL in the (XML based) SVN < 1.4 entries file"); - processURL(message, depth, svnEntryCopyFromUrl, baseURL); - } - } - } else { - // text based format us being used, so >= SVN 1.4, and < SVN 1.7.x - // Parse each line in the ".svn/entries" file - // we cannot use the StringTokenizer approach used by the robots.txt logic, - // since this causes empty lines to be ignored, which causes problems... - String previousline = null; - String[] lines = content.split("\n"); - for (String line : lines) { - // If the line is empty, skip it - if (line.length() > 0) { - - // getLogger().debug("Processing SVN entries line: " + line); - - Matcher matcher = svnTextFormatFileOrDirectoryPattern.matcher(line); - if (matcher.find()) { - // filetype is "dir" or "file", as per the contents of the SVN file. - String filetype = matcher.group(0); - // the previous line actually contains the file/directory name. - if (previousline != null && previousline.length() > 0) { - getLogger() - .debug( - "Found a file/directory name in the (text based) SVN 1.4/1.5/1.6 SVN entries file"); - - processURL( - message, - depth, - "../" + previousline + (filetype.equals("dir") ? "/" : ""), - baseURL); - // get the internal SVN file, probably leading to source code disclosure - if (filetype.equals("file")) { - processURL( - message, - depth, - "text-base/" + previousline + ".svn-base", - baseURL); - } - - // re-seed the spider for this directory. - if (filetype.equals("dir")) { - processURL( - message, - depth, - "../" + previousline + "/.svn/entries", - baseURL); - } - } - } else { - // not a "file" or "dir" line, but it may contain details of the SVN repo - // location - Matcher repoMatcher = svnRepoLocationPattern.matcher(line); - if (repoMatcher.find()) { - getLogger() - .debug( - "Found an SVN repository location in the (text based) 1.4/1.5/1.6 SVN entries file"); - - processURL(message, depth, line + "/", baseURL); - } - } - } - // last thing to do is to record the line as the previous line for the next - // iteration. - previousline = line; - } - } - // We consider the message fully parsed, so it doesn't get parsed by 'fallback' parsers - return true; - } - - @Override - public boolean canParseResource(HttpMessage message, String path, boolean wasAlreadyParsed) { - // matches the file name of files that should be parsed with the SVN entries file parser - Matcher matcher = SVN_ENTRIES_FILE_PATTERN.matcher(path); - return matcher.find(); - } -} diff --git a/zap/src/main/java/org/zaproxy/zap/spider/parser/SpiderSitemapXMLParser.java b/zap/src/main/java/org/zaproxy/zap/spider/parser/SpiderSitemapXMLParser.java deleted file mode 100644 index 1f41af64c8e..00000000000 --- a/zap/src/main/java/org/zaproxy/zap/spider/parser/SpiderSitemapXMLParser.java +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2014 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.spider.parser; - -import java.io.ByteArrayInputStream; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.ParserConfigurationException; -import javax.xml.xpath.XPath; -import javax.xml.xpath.XPathConstants; -import javax.xml.xpath.XPathExpression; -import javax.xml.xpath.XPathExpressionException; -import javax.xml.xpath.XPathFactory; -import net.htmlparser.jericho.Source; -import org.apache.logging.log4j.LogManager; -import org.parosproxy.paros.network.HttpMessage; -import org.parosproxy.paros.network.HttpStatusCode; -import org.w3c.dom.Document; -import org.w3c.dom.NodeList; -import org.xml.sax.InputSource; -import org.zaproxy.zap.utils.XmlUtils; - -/** - * SitemapXMLParser is used for parsing URLs from a sitemap.xml file, which sometimes (very - * helpfully) resides in the web root. - * - * @author 70pointer - * @deprecated (2.12.0) See the spider add-on in zap-extensions instead. - */ -@Deprecated -public class SpiderSitemapXMLParser extends SpiderParser { - - /** a pattern to match the sitemap.xml file name */ - private Pattern SITEMAP_XML_FILENAME_PATTERN = Pattern.compile("/sitemap\\.xml$"); - - /** a pattern to match the sitemap.xml file.. hint: It's XML */ - private static final Pattern xmlPattern = - Pattern.compile( - "^<\\?xml\\s+version\\s*=\\s*\"[0-9.]+\"\\s+encoding\\s*=\\s*\"[^\"]+\"\\s*\\?>"); - - /** used to parse the XML based file format */ - private static DocumentBuilder dBuilder; - - /** an x path expression to match the "loc" tag in sitemap.xml */ - private static XPathExpression xpathLocationExpression; - - /** statically initialise the XML DocumentBuilderFactory and DocumentBuilder */ - static { - try { - dBuilder = XmlUtils.newXxeDisabledDocumentBuilderFactory().newDocumentBuilder(); - XPath xpath = XPathFactory.newInstance().newXPath(); - xpathLocationExpression = xpath.compile("/urlset/url/loc/text()"); - } catch (ParserConfigurationException | XPathExpressionException e) { - LogManager.getLogger(SpiderSitemapXMLParser.class).error(e); - } - } - - /** - * Instantiates a new sitemap.xml parser. - * - * @param params the params - * @throws NullPointerException if {@code params} is null. - */ - public SpiderSitemapXMLParser(org.zaproxy.zap.spider.SpiderParam params) { - super(params); - } - - @Override - public boolean parseResource(HttpMessage message, Source source, int depth) { - - getLogger().debug("Parsing a sitemap.xml resource..."); - - if (message == null - || !getSpiderParam().isParseSitemapXml() - || !message.getResponseHeader().isXml() - || HttpStatusCode.isClientError(message.getResponseHeader().getStatusCode()) - || HttpStatusCode.isServerError(message.getResponseHeader().getStatusCode())) { - return false; - } - - // Get the response content - byte[] response = message.getResponseBody().getBytes(); - String baseURL = message.getRequestHeader().getURI().toString(); - Matcher xmlFormatMatcher = xmlPattern.matcher(new String(response)); - if (xmlFormatMatcher.find()) { - - getLogger().debug("The format matches XML"); - - try { - Document xmldoc = - dBuilder.parse(new InputSource(new ByteArrayInputStream(response))); - NodeList locationNodes = - (NodeList) xpathLocationExpression.evaluate(xmldoc, XPathConstants.NODESET); - for (int i = 0; i < locationNodes.getLength(); i++) { - processURL(message, depth, locationNodes.item(i).getNodeValue(), baseURL); - } - } catch (Exception e) { - getLogger().error("An error occurred trying to parse sitemap.xml", e); - return false; - } - // We consider the message fully parsed, so it doesn't get parsed by 'fallback' parsers - return true; - } else { - // the file name is right, but the content is not. Pass it to another parser. - getLogger() - .debug( - "The content of the response from '{}' does not match the expected content for a sitemap.xml file. Ignoring it.", - baseURL); - return false; - } - } - - @Override - public boolean canParseResource(HttpMessage message, String path, boolean wasAlreadyParsed) { - getLogger().debug("canParseResource called on '{}'", path); - // matches the file name of files that should be parsed with the sitemap.xml file parser - Matcher matcher = SITEMAP_XML_FILENAME_PATTERN.matcher(path); - return matcher.find(); - } -} diff --git a/zap/src/main/java/org/zaproxy/zap/spider/parser/SpiderTextParser.java b/zap/src/main/java/org/zaproxy/zap/spider/parser/SpiderTextParser.java deleted file mode 100644 index 991532039db..00000000000 --- a/zap/src/main/java/org/zaproxy/zap/spider/parser/SpiderTextParser.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2012 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.spider.parser; - -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import net.htmlparser.jericho.Source; -import org.parosproxy.paros.network.HttpMessage; - -/** - * The Class SpiderTextParser is used for parsing of simple text (non-HTML) files, gathering - * resource urls from them. For example it is used for parsing CSS, less, javascript files, - * searching for urls. - * - * @deprecated (2.12.0) See the spider add-on in zap-extensions instead. - */ -@Deprecated -public class SpiderTextParser extends SpiderParser { - - /** The Constant urlPattern defining the pattern for an url. */ - private static final Pattern patternURL = - Pattern.compile( - "\\W(http(s?)://[^\\x00-\\x1f\"'\\s<>#()\\[\\]{}]+)", Pattern.CASE_INSENSITIVE); - - public SpiderTextParser() { - this(null); - } - - public SpiderTextParser(org.zaproxy.zap.spider.SpiderParam params) { - super(params); - } - - @Override - public boolean parseResource(HttpMessage message, Source source, int depth) { - getLogger().debug("Parsing a non-HTML text resource."); - - String baseURL = message.getRequestHeader().getURI().toString(); - - // Use a simple pattern matcher to find urls - Matcher matcher = patternURL.matcher(message.getResponseBody().toString()); - while (matcher.find()) { - String s = matcher.group(1); - processURL(message, depth, s, baseURL); - } - - return false; - } - - @Override - public boolean canParseResource(HttpMessage message, String path, boolean wasAlreadyConsumed) { - // Fall-back parser - if it's a text, non-HTML response which has not already been processed - return !wasAlreadyConsumed - && message.getResponseHeader().isText() - && !message.getResponseHeader().isHtml(); - } -} diff --git a/zap/src/main/java/org/zaproxy/zap/view/popup/PopupMenuHttpMessageContainer.java b/zap/src/main/java/org/zaproxy/zap/view/popup/PopupMenuHttpMessageContainer.java index aa53204737c..77e5e2feec8 100644 --- a/zap/src/main/java/org/zaproxy/zap/view/popup/PopupMenuHttpMessageContainer.java +++ b/zap/src/main/java/org/zaproxy/zap/view/popup/PopupMenuHttpMessageContainer.java @@ -350,7 +350,6 @@ protected boolean isEnable(HttpMessageContainer httpMessageContainer) { * @return the invoker or {@code Invoker#UNKNOWN} if the message container was not identified. * @see Invoker */ - @SuppressWarnings("deprecation") protected static final Invoker getInvoker(HttpMessageContainer httpMessageContainer) { Invoker invoker; switch (httpMessageContainer.getName()) { @@ -366,7 +365,7 @@ protected static final Invoker getInvoker(HttpMessageContainer httpMessageContai case SearchPanel.HTTP_MESSAGE_CONTAINER_NAME: invoker = Invoker.SEARCH_PANEL; break; - case org.zaproxy.zap.extension.spider.SpiderPanel.HTTP_MESSAGE_CONTAINER_NAME: + case "SpiderHttpMessageContainer": invoker = Invoker.SPIDER_PANEL; break; case ActiveScanPanel.MESSAGE_CONTAINER_NAME: diff --git a/zap/src/main/java/org/zaproxy/zap/view/popup/PopupMenuItemHttpMessageContainer.java b/zap/src/main/java/org/zaproxy/zap/view/popup/PopupMenuItemHttpMessageContainer.java index 4770c9f63a2..34ce1c61c57 100644 --- a/zap/src/main/java/org/zaproxy/zap/view/popup/PopupMenuItemHttpMessageContainer.java +++ b/zap/src/main/java/org/zaproxy/zap/view/popup/PopupMenuItemHttpMessageContainer.java @@ -269,7 +269,6 @@ protected boolean isEnable(HttpMessageContainer httpMessageContainer) { * @return the invoker or {@code Invoker#UNKNOWN} if the message container was not identified. * @see Invoker */ - @SuppressWarnings("deprecation") private static Invoker getInvoker(HttpMessageContainer httpMessageContainer) { Invoker invoker; switch (httpMessageContainer.getName()) { @@ -286,7 +285,7 @@ private static Invoker getInvoker(HttpMessageContainer httpMessageContainer) { case SearchPanel.HTTP_MESSAGE_CONTAINER_NAME: invoker = Invoker.SEARCH_PANEL; break; - case org.zaproxy.zap.extension.spider.SpiderPanel.HTTP_MESSAGE_CONTAINER_NAME: + case "SpiderHttpMessageContainer": invoker = Invoker.SPIDER_PANEL; break; case ActiveScanPanel.MESSAGE_CONTAINER_NAME: diff --git a/zap/src/main/resources/org/zaproxy/zap/resources/Messages.properties b/zap/src/main/resources/org/zaproxy/zap/resources/Messages.properties index 6971760c8f0..308583173c6 100644 --- a/zap/src/main/resources/org/zaproxy/zap/resources/Messages.properties +++ b/zap/src/main/resources/org/zaproxy/zap/resources/Messages.properties @@ -918,17 +918,6 @@ callback.panel.table.column.handler = Handler callback.panel.table.column.referer = Referer callback.test.msg = Received test callback: {0} from {1} -certificates.pkcs11.drivers.button.add = Add -certificates.pkcs11.drivers.button.browse = Browse -certificates.pkcs11.drivers.button.close = Close -certificates.pkcs11.drivers.button.delete = Delete -certificates.pkcs11.drivers.label.name = Name -certificates.pkcs11.drivers.label.path = Path (Path to a DLL for PKCS#11 support - usually delivered with the smartcard software) -certificates.pkcs11.drivers.label.slot = Slot -certificates.pkcs11.drivers.label.slotIndex = Slot Index -certificates.pkcs11.drivers.title = PKCS#11 Drivers -certificates.pkcs11.label.experimentalSliSupport = Enable experimental Slot List Index support - cfu.button.addons.browse = Manage Add-ons cfu.button.addons.download = Download Selected cfu.button.addons.info = More Info @@ -1971,7 +1960,6 @@ menu.analyse.mnemonic = a menu.analyse.scan = Scan menu.analyse.scanAll = Scan All menu.analyse.scanPolicy = Scan Policy Manager... -menu.analyse.spider = Spider... menu.edit = Edit menu.edit.find = Find... menu.edit.find.mnemonic = f @@ -2064,7 +2052,6 @@ menu.tools.mnemonic = t menu.tools.options = Options... menu.tools.options.errorSavingOptions = Error saving options. menu.tools.options.mnemonic = o -menu.tools.spider = Spider... menu.view = View menu.view.enableImage = Enable Image in History menu.view.mnemonic = v @@ -2150,40 +2137,6 @@ options.acsrf.label.tokens =

These tokens are treated as anti-CSR options.acsrf.table.header.enabled = Enabled options.acsrf.table.header.token = Token options.acsrf.title = Anti-CSRF Tokens -options.cert.button.browse = Browse -options.cert.button.delete = Delete -options.cert.button.keystore = Add to KeyStore -options.cert.button.pkcs11 = Add to KeyStore -options.cert.button.setactive = Set Active -options.cert.error = Error -options.cert.error.accesskeystore = Error accessing KeyStore: -options.cert.error.fingerprint = Error calculating key fingerprint: -options.cert.error.password = Maybe your password or driver is wrong. -options.cert.error.password.blank = You've left the PIN field blank. -options.cert.error.pkcs11 = Try to add the PKCS#11 driver again... -options.cert.error.pkcs11notavailable =

The required Sun/IBM PKCS#11 provider is not available.

For more information visit the pages:

-options.cert.error.pkcs11notavailable.ibm.hyperlink = http://pic.dhe.ibm.com/infocenter/java7sdk/v7r0/topic/com.ibm.java.security.component.71.doc/security-component/pkcs11implDocs/ibmpkcs11.html -options.cert.error.pkcs11notavailable.ibm.hyperlink.text = IBMPKCS11Impl Provider -options.cert.error.pkcs11notavailable.sun.hyperlink = http://docs.oracle.com/javase/7/docs/technotes/guides/security/p11guide.html#Requirements -options.cert.error.pkcs11notavailable.sun.hyperlink.text = Sun PKCS#11 Provider -options.cert.error.pkcs12nopass = PKCS#12 files with empty passwords are not supported. -options.cert.error.usepassfile = Please use a password protected file. -options.cert.error.wrongpassword = Incorrect PKCS#11 PIN or password -options.cert.error.wrongpasswordlast = (Last try before blocking the smartcard) -options.cert.label.activecerts = Active Certificate -options.cert.label.addkeystore =

Add your keystore and select the desired certificate.

Certificate setting will not be stored in options and you will need to enable certificate next time you restart ZAP.

-options.cert.label.client.cert = Client Certificate -options.cert.label.driver = Driver -options.cert.label.enterpassword = Enter Password -options.cert.label.file = File -options.cert.label.keystore = KeyStore -options.cert.label.password = Password -options.cert.label.pincode = PIN Code -options.cert.label.useclientcert = Use client certificate -options.cert.tab.keystore = KeyStore -options.cert.tab.pkcs = PKCS#12 -options.cert.tab.pkcs11 = PKCS#11 -options.cert.title = Client Certificate options.dialog.reset.button = Reset to Factory Defaults options.dialog.reset.error = Failed to reset the options:\n{0} options.dialog.reset.error.panel = Failed to reset {0} options panel:\n{1} @@ -2762,251 +2715,10 @@ sites.purge.popup = Delete sites.purge.title = Delete Site Node(s) sites.purge.warning = Are you sure you want to delete the node(s)?\nThis will delete all of the messages associated with the deleted Site Tree nodes. sites.showinsites.popup = Show in Sites Tab -sites.spider.popup = Spider... siteselect.button.select = Select siteselect.dialog.title = Select Node -spider.activeActionPrefix = Spidering: {0} -spider.api.action.addDomainAlwaysInScope = Adds a new domain that's always in scope, using the specified value. Optionally sets if the new entry is enabled (default, true) and whether or not the new value is specified as a regex (default, false). -spider.api.action.addDomainAlwaysInScope.param.isEnabled = -spider.api.action.addDomainAlwaysInScope.param.isRegex = -spider.api.action.addDomainAlwaysInScope.param.value = -spider.api.action.clearExcludedFromScan = Clears the regexes of URLs excluded from the spider scans. -spider.api.action.disableAllDomainsAlwaysInScope = Disables all domains that are always in scope. -spider.api.action.enableAllDomainsAlwaysInScope = Enables all domains that are always in scope. -spider.api.action.excludeFromScan = Adds a regex of URLs that should be excluded from the spider scans. -spider.api.action.excludeFromScan.param.regex = -spider.api.action.modifyDomainAlwaysInScope = Modifies a domain that's always in scope. Allows to modify the value, if enabled or if a regex. The domain is selected with its index, which can be obtained with the view domainsAlwaysInScope. -spider.api.action.modifyDomainAlwaysInScope.param.idx = -spider.api.action.modifyDomainAlwaysInScope.param.isEnabled = -spider.api.action.modifyDomainAlwaysInScope.param.isRegex = -spider.api.action.modifyDomainAlwaysInScope.param.value = -spider.api.action.pause = -spider.api.action.pause.param.scanId = -spider.api.action.pauseAllScans = -spider.api.action.removeAllScans = -spider.api.action.removeDomainAlwaysInScope = Removes a domain that's always in scope, with the given index. The index can be obtained with the view domainsAlwaysInScope. -spider.api.action.removeDomainAlwaysInScope.param.idx = -spider.api.action.removeScan = -spider.api.action.removeScan.param.scanId = -spider.api.action.resume = -spider.api.action.resume.param.scanId = -spider.api.action.resumeAllScans = -spider.api.action.scan = Runs the spider against the given URL (or context). Optionally, the 'maxChildren' parameter can be set to limit the number of children scanned, the 'recurse' parameter can be used to prevent the spider from seeding recursively, the parameter 'contextName' can be used to constrain the scan to a Context and the parameter 'subtreeOnly' allows to restrict the spider under a site's subtree (using the specified 'url'). -spider.api.action.scan.param.contextName = -spider.api.action.scan.param.maxChildren = -spider.api.action.scan.param.recurse = -spider.api.action.scan.param.subtreeOnly = -spider.api.action.scan.param.url = -spider.api.action.scanAsUser = Runs the spider from the perspective of a User, obtained using the given Context ID and User ID. See 'scan' action for more details. -spider.api.action.scanAsUser.param.contextId = -spider.api.action.scanAsUser.param.maxChildren = -spider.api.action.scanAsUser.param.recurse = -spider.api.action.scanAsUser.param.subtreeOnly = -spider.api.action.scanAsUser.param.url = -spider.api.action.scanAsUser.param.userId = -spider.api.action.setOptionAcceptCookies = Sets whether or not a spider process should accept cookies while spidering. -spider.api.action.setOptionAcceptCookies.param.Boolean = -spider.api.action.setOptionHandleODataParametersVisited = -spider.api.action.setOptionHandleODataParametersVisited.param.Boolean = -spider.api.action.setOptionHandleParameters = -spider.api.action.setOptionHandleParameters.param.String = -spider.api.action.setOptionMaxChildren = Sets the maximum number of child nodes (per node) that can be crawled, 0 means no limit. -spider.api.action.setOptionMaxChildren.param.Integer = -spider.api.action.setOptionMaxDepth = Sets the maximum depth the spider can crawl, 0 for unlimited depth. -spider.api.action.setOptionMaxDepth.param.Integer = -spider.api.action.setOptionMaxDuration = -spider.api.action.setOptionMaxDuration.param.Integer = -spider.api.action.setOptionMaxParseSizeBytes = Sets the maximum size, in bytes, that a response might have to be parsed. This allows the spider to skip big responses/files. -spider.api.action.setOptionMaxParseSizeBytes.param.Integer = -spider.api.action.setOptionMaxScansInUI = -spider.api.action.setOptionMaxScansInUI.param.Integer = -spider.api.action.setOptionParseComments = -spider.api.action.setOptionParseComments.param.Boolean = -spider.api.action.setOptionParseGit = -spider.api.action.setOptionParseGit.param.Boolean = -spider.api.action.setOptionParseRobotsTxt = -spider.api.action.setOptionParseRobotsTxt.param.Boolean = -spider.api.action.setOptionParseSVNEntries = -spider.api.action.setOptionParseSVNEntries.param.Boolean = -spider.api.action.setOptionParseSitemapXml = -spider.api.action.setOptionParseSitemapXml.param.Boolean = -spider.api.action.setOptionPostForm = -spider.api.action.setOptionPostForm.param.Boolean = -spider.api.action.setOptionProcessForm = -spider.api.action.setOptionProcessForm.param.Boolean = -spider.api.action.setOptionRequestWaitTime = -spider.api.action.setOptionRequestWaitTime.param.Integer = -spider.api.action.setOptionScopeString = Use actions [add|modify|remove]DomainAlwaysInScope instead. -spider.api.action.setOptionScopeString.param.String = -spider.api.action.setOptionSendRefererHeader = Sets whether or not the 'Referer' header should be sent while spidering. -spider.api.action.setOptionSendRefererHeader.param.Boolean = -spider.api.action.setOptionShowAdvancedDialog = -spider.api.action.setOptionShowAdvancedDialog.param.Boolean = -spider.api.action.setOptionSkipURLString = -spider.api.action.setOptionSkipURLString.param.String = -spider.api.action.setOptionThreadCount = -spider.api.action.setOptionThreadCount.param.Integer = -spider.api.action.setOptionUserAgent = -spider.api.action.setOptionUserAgent.param.String = -spider.api.action.stop = -spider.api.action.stop.param.scanId = -spider.api.action.stopAllScans = -spider.api.desc = -spider.api.view.addedNodes = Returns a list of the names of the nodes added to the Sites tree by the specified scan. -spider.api.view.addedNodes.param.scanId = -spider.api.view.allUrls = Returns a list of unique URLs from the history table based on HTTP messages added by the Spider. -spider.api.view.domainsAlwaysInScope = Gets all the domains that are always in scope. For each domain the following are shown: the index, the value (domain), if enabled, and if specified as a regex. -spider.api.view.excludedFromScan = Gets the regexes of URLs excluded from the spider scans. -spider.api.view.fullResults = -spider.api.view.fullResults.param.scanId = -spider.api.view.optionAcceptCookies = Gets whether or not a spider process should accept cookies while spidering. -spider.api.view.optionDomainsAlwaysInScope = Use view domainsAlwaysInScope instead. -spider.api.view.optionDomainsAlwaysInScopeEnabled = Use view domainsAlwaysInScope instead. -spider.api.view.optionHandleODataParametersVisited = -spider.api.view.optionHandleParameters = -spider.api.view.optionMaxChildren = Gets the maximum number of child nodes (per node) that can be crawled, 0 means no limit. -spider.api.view.optionMaxDepth = Gets the maximum depth the spider can crawl, 0 if unlimited. -spider.api.view.optionMaxDuration = -spider.api.view.optionMaxParseSizeBytes = Gets the maximum size, in bytes, that a response might have to be parsed. -spider.api.view.optionMaxScansInUI = -spider.api.view.optionParseComments = -spider.api.view.optionParseGit = -spider.api.view.optionParseRobotsTxt = -spider.api.view.optionParseSVNEntries = -spider.api.view.optionParseSitemapXml = -spider.api.view.optionPostForm = -spider.api.view.optionProcessForm = -spider.api.view.optionRequestWaitTime = -spider.api.view.optionScope = -spider.api.view.optionScopeText = -spider.api.view.optionSendRefererHeader = Gets whether or not the 'Referer' header should be sent while spidering. -spider.api.view.optionShowAdvancedDialog = -spider.api.view.optionSkipURLString = -spider.api.view.optionThreadCount = -spider.api.view.optionUserAgent = -spider.api.view.results = -spider.api.view.results.param.scanId = -spider.api.view.scans = -spider.api.view.status = -spider.api.view.status.param.scanId = -spider.context.popup = Spider Context... -spider.context.user.popup = Spider Context as User... -spider.custom.button.reset = Reset -spider.custom.button.scan = Start Scan -spider.custom.label.acceptcookies = Accept Cookies: -spider.custom.label.adv = Show Advanced Options -spider.custom.label.context = Context: -spider.custom.label.handleOdata = Handle OData Parameters: -spider.custom.label.irrelevantUrlParameters = Irrelevant URL Parameters: -spider.custom.label.maxChildren = Maximum Children to Crawl (0 is unlimited): -spider.custom.label.maxDepth = Maximum Depth to Crawl (0 is unlimited): -spider.custom.label.maxDuration = Maximum Duration (minutes; 0 is unlimited): -spider.custom.label.maxParseSizeBytes = Maximum Parse Size (bytes): -spider.custom.label.parseComments = Parse HTML Comments: -spider.custom.label.parseGit = Parse Git Metadata: -spider.custom.label.parseRobots = Parse 'robots.txt': -spider.custom.label.parseSvn = Parse SVN Metadata: -spider.custom.label.postForms = POST Forms: -spider.custom.label.processForms = Process Forms: -spider.custom.label.recurse = Recurse: -spider.custom.label.sendReferer = Send 'Referer' Header: -spider.custom.label.sitemap = Parse 'sitemap.xml': -spider.custom.label.spiderSubtreeOnly = Spider Subtree Only -spider.custom.label.start = Starting Point: -spider.custom.label.user = User: -spider.custom.noStartSubtreeOnly.error = A site node must be selected or a URL manually introduced, to spider a site's subtree. -spider.custom.nostart.error = You must select a valid starting point\nincluding the protocol e.g. https://www.example.com -spider.custom.notSafe.error = Spider scans are not allowed in 'Safe' mode. -spider.custom.popup = Spider... -spider.custom.tab.adv = Advanced -spider.custom.tab.scope = Scope -spider.custom.targetNotInScope.error = The following target is not allowed in 'Protected' mode:\n{0} -spider.custom.title = Spider -spider.desc = Spider used for automatically finding URIs on a site -spider.label.inScope = URI found during crawl: -spider.label.outOfScope = URI found but out of crawl scope: -spider.name = Spider Extension -spider.options.domains.in.scope.add.button.confirm = Add -spider.options.domains.in.scope.add.title = Add Domain Always In Scope -spider.options.domains.in.scope.dialog.remove.button.cancel = Cancel -spider.options.domains.in.scope.dialog.remove.button.confirm = Remove -spider.options.domains.in.scope.dialog.remove.checkbox.label = Do not show this message again. -spider.options.domains.in.scope.dialog.remove.text = Are you sure you want to remove the selected domain? -spider.options.domains.in.scope.dialog.remove.title = Remove Domain Always In Scope -spider.options.domains.in.scope.field.label.domain = Domain: -spider.options.domains.in.scope.field.label.enabled = Enabled: -spider.options.domains.in.scope.field.label.regex = Regex: -spider.options.domains.in.scope.modify.button.confirm = Modify -spider.options.domains.in.scope.modify.title = Modify Domain Always In Scope -spider.options.domains.in.scope.table.header.enabled = Enabled -spider.options.domains.in.scope.table.header.regex = Regex -spider.options.domains.in.scope.table.header.value = Domain -spider.options.domains.in.scope.warning.invalid.regex.text = The regular expression is invalid. -spider.options.domains.in.scope.warning.invalid.regex.title = Domain Always In Scope Regex Invalid -spider.options.label.acceptcookies = Accept Cookies -spider.options.label.comments = Parse HTML Comments -spider.options.label.depth = Maximum Depth to Crawl (0 is unlimited): -spider.options.label.domains = Domains that are always 'in scope' -spider.options.label.duration = Maximum Duration (minutes; 0 is unlimited): -spider.options.label.git = Parse Git metadata files for new URIs -spider.options.label.handlehodataparameters = Handle OData-specific parameters -spider.options.label.handleparameters = Query parameters handling for checking visited URIs: -spider.options.label.irrelevantUrlParameters = Irrelevant URL parameters -spider.options.label.maxChildren = Maximum Children to Crawl (0 is unlimited): -spider.options.label.maxParseSizeBytes = Maximum Parse Size (bytes): -spider.options.label.post = POST forms (recommended but may generate unwanted requests) -spider.options.label.processform = Process forms (forms are processed and GET queries submitted) -spider.options.label.robotstxt = Parse 'robots.txt' files for new URIs -spider.options.label.sendRefererHeader = Send "Referer" header -spider.options.label.sitemapxml = Parse 'sitemap.xml' files for new URIs -spider.options.label.svnentries = Parse SVN metadata files for new URIs -spider.options.label.threads = Number of Threads Used: -spider.options.title = Spider -spider.options.value.handleparameters.ignoreAll = Ignore parameters completely -spider.options.value.handleparameters.ignoreValue = Consider only parameter's name -spider.options.value.handleparameters.useAll = Consider both parameter's name and value -spider.panel.emptyView = You need to visit the website via a browser first and select a URL/folder/node in the 'Sites' panel displayed. -spider.panel.mnemonic = d -spider.panel.tab.addednodes = Added Nodes -spider.panel.tab.messages = Messages -spider.panel.tab.urls = URLs -spider.panel.title = Spider -spider.parsefilter.reason.empty = Empty Message -spider.parsefilter.reason.maxchildren = Max Children -spider.parsefilter.reason.maxsize = Max Size -spider.parsefilter.reason.nottext = Not Text -spider.site.popup = Spider Site -spider.subtree.popup = Spider Subtree -spider.table.flags.illegalprotocol = Illegal Protocol -spider.table.flags.outofcontext = Out of Context -spider.table.flags.outofscope = Out of Scope -spider.table.flags.seed = Seed -spider.table.flags.userrules = User Rules -spider.table.header.flags = Flags -spider.table.header.inScope = Processed -spider.table.header.method = Method -spider.table.header.uri = URI -spider.table.messages.column.processed.successfully = Successfully -spider.table.messages.header.processed = Processed -spider.task.message.skipped.ioerror = I/O Error -spider.task.message.skipped.maxdepth = Max Depth -spider.task.message.skipped.stopped = Spider Stopped -spider.toolbar.added.label = Nodes Added: -spider.toolbar.ascans.label = Current Scans: -spider.toolbar.button.clear = Clean completed scans -spider.toolbar.button.new = New Scan -spider.toolbar.button.options = Spider Options -spider.toolbar.button.pause = Pause Spider -spider.toolbar.button.stop = Stop Spider -spider.toolbar.button.unpause = Resume Spider -spider.toolbar.found.label = URLs Found: -spider.toolbar.progress.label = Progress: -spider.toolbar.progress.select = --Select Scan-- -spider.url.popup = Spider URL -spider.url.user.popup = Spider URL as User... - start.cmdline.badfile = File type is not supported ''{0}'' start.cmdline.badparam = Unsupported option ''{0}''.; start.cmdline.nofile = File not found ''{0}'' @@ -3263,8 +2975,6 @@ variant.shortname.urlpath = URL Path variant.shortname.userdefined = User Defined variant.shortname.xml = XML Tag/Attribute -view.cert.button.close = Close -view.cert.title = Certificate view.dialog.dontPrompt = Do not show this message again. view.dialog.remember = Remember my choice and do not show this message again. view.href.table.cell.alert.risk.label.high = High diff --git a/zap/src/test/java/org/zaproxy/zap/spider/SpiderControllerUnitTest.java b/zap/src/test/java/org/zaproxy/zap/spider/SpiderControllerUnitTest.java deleted file mode 100644 index 4156f001abc..00000000000 --- a/zap/src/test/java/org/zaproxy/zap/spider/SpiderControllerUnitTest.java +++ /dev/null @@ -1,314 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2021 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.spider; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.BDDMockito.given; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.parosproxy.paros.db.RecordHistory; -import org.parosproxy.paros.db.TableAlert; -import org.parosproxy.paros.db.TableHistory; -import org.parosproxy.paros.model.HistoryReference; -import org.parosproxy.paros.model.Model; -import org.parosproxy.paros.network.HttpHeaderField; -import org.parosproxy.paros.network.HttpRequestHeader; -import org.zaproxy.zap.WithConfigsTest; - -/** Unit test for {@link SpiderController}. */ -@SuppressWarnings({"deprecation", "removal"}) -class SpiderControllerUnitTest extends WithConfigsTest { - - private org.zaproxy.zap.spider.Spider spider; - private org.zaproxy.zap.spider.SpiderController spiderController; - - @BeforeAll - static void setUpTables() throws Exception { - TableHistory tableHistory = mock(TableHistory.class); - given(tableHistory.write(anyLong(), anyInt(), any())).willReturn(mock(RecordHistory.class)); - HistoryReference.setTableHistory(tableHistory); - HistoryReference.setTableAlert(mock(TableAlert.class)); - } - - @AfterAll - static void cleanUpTables() { - HistoryReference.setTableHistory(null); - HistoryReference.setTableAlert(null); - } - - @BeforeEach - void setUp() { - spider = mock(org.zaproxy.zap.spider.Spider.class); - - given(spider.getSpiderParam()).willReturn(new org.zaproxy.zap.spider.SpiderParam()); - given(spider.getModel()).willReturn(Model.getSingleton()); - - org.zaproxy.zap.extension.spider.ExtensionSpider extensionSpider = - mock(org.zaproxy.zap.extension.spider.ExtensionSpider.class); - given(extensionSpider.getValueGenerator()) - .willReturn(mock(org.zaproxy.zap.model.ValueGenerator.class)); - given(spider.getExtensionSpider()).willReturn(extensionSpider); - - spiderController = - new org.zaproxy.zap.spider.SpiderController(spider, Collections.emptyList()); - } - - @Test - void shouldSubmitTasksForDifferentSpiderResources() { - // Given - List requestHeaders = new ArrayList<>(); - requestHeaders.add(new HttpHeaderField("X-Custom-Header-1", "xyz")); - // When - spiderController.resourceFound( - createBasicGetSpiderResourceFound("https://example.com/test.html", 1)); - spiderController.resourceFound( - createBasicGetSpiderResourceFound("https://example.com/test.html/", 1)); - spiderController.resourceFound( - createGetSpiderResourceFoundWithHeaders( - "https://example.com/test.html", 1, false, requestHeaders)); - spiderController.resourceFound( - createBasicPostSpiderResourceFound("https://example.com/test.html", "", 1)); - spiderController.resourceFound( - createBasicPostSpiderResourceFound("https://example.com/test.html", "A=1", 1)); - spiderController.resourceFound( - createBasicPostSpiderResourceFound("https://example.com/test.html", "A=2", 1)); - spiderController.resourceFound( - createPostSpiderResourceFoundWithHeaders( - "https://example.com/test.html", "A=2", 1, false, requestHeaders)); - // Then - verify(spider, times(7)).submitTask(any()); - } - - @Test - void shouldSubmitTasksForDifferentMethods() { - // Given - org.zaproxy.zap.spider.parser.SpiderResourceFound getResource = - org.zaproxy.zap.spider.parser.SpiderResourceFound.builder() - .setMethod(HttpRequestHeader.GET) - .setUri("http://test.com") - .build(); - org.zaproxy.zap.spider.parser.SpiderResourceFound postResource = - org.zaproxy.zap.spider.parser.SpiderResourceFound.builder() - .setMethod(HttpRequestHeader.POST) - .setUri("http://test.com") - .build(); - org.zaproxy.zap.spider.parser.SpiderResourceFound putResource = - org.zaproxy.zap.spider.parser.SpiderResourceFound.builder() - .setMethod(HttpRequestHeader.PUT) - .setUri("http://test.com") - .build(); - org.zaproxy.zap.spider.parser.SpiderResourceFound deleteResource = - org.zaproxy.zap.spider.parser.SpiderResourceFound.builder() - .setMethod(HttpRequestHeader.DELETE) - .setUri("http://test.com") - .build(); - org.zaproxy.zap.spider.parser.SpiderResourceFound headResource = - org.zaproxy.zap.spider.parser.SpiderResourceFound.builder() - .setMethod(HttpRequestHeader.HEAD) - .setUri("http://test.com") - .build(); - // When - spiderController.resourceFound(getResource); - spiderController.resourceFound(postResource); - spiderController.resourceFound(putResource); - spiderController.resourceFound(deleteResource); - spiderController.resourceFound(headResource); - // Then - verify(spider, times(5)).submitTask(any()); - } - - @Test - void shouldNotSubmitSameGetTaskWithDifferentDepthAndIgnore() { - // Given - org.zaproxy.zap.spider.parser.SpiderResourceFound spiderResourceFoundDepth1 = - createGetSpiderResourceFoundWithHeaders( - "https://example.com/test.html", 1, false, Collections.emptyList()); - org.zaproxy.zap.spider.parser.SpiderResourceFound spiderResourceFoundDepth2Ignore = - createGetSpiderResourceFoundWithHeaders( - "https://example.com/test.html", 2, true, Collections.emptyList()); - // When - spiderController.resourceFound(spiderResourceFoundDepth1); - spiderController.resourceFound(spiderResourceFoundDepth1); - spiderController.resourceFound(spiderResourceFoundDepth2Ignore); - // Then - verify(spider).submitTask(any()); - } - - @Test - void shouldNotSubmitSamePostTaskWithDifferentDepthAndIgnore() { - // Given - org.zaproxy.zap.spider.parser.SpiderResourceFound spiderResourceFoundDepth1 = - createPostSpiderResourceFoundWithHeaders( - "https://example.com/test.html", "body", 1, false, Collections.emptyList()); - org.zaproxy.zap.spider.parser.SpiderResourceFound spiderResourceFoundDepth2Ignore = - createPostSpiderResourceFoundWithHeaders( - "https://example.com/test.html", "body", 2, true, Collections.emptyList()); - // When - spiderController.resourceFound(spiderResourceFoundDepth1); - spiderController.resourceFound(spiderResourceFoundDepth1); - spiderController.resourceFound(spiderResourceFoundDepth2Ignore); - // Then - verify(spider).submitTask(any()); - } - - @Test - void shouldNotSubmitSameGetTaskWithDifferentHeaderOrder() { - // Given - List requestHeadersOrder1 = new ArrayList<>(); - requestHeadersOrder1.add(new HttpHeaderField("X-Custom-Header-1", "xyz")); - requestHeadersOrder1.add(new HttpHeaderField("X-Custom-Header-2", "123")); - List requestHeadersOrder2 = new ArrayList<>(); - requestHeadersOrder2.add(new HttpHeaderField("X-Custom-Header-2", "123")); - requestHeadersOrder2.add(new HttpHeaderField("X-Custom-Header-1", "xyz")); - org.zaproxy.zap.spider.parser.SpiderResourceFound spiderResourceFound1 = - createGetSpiderResourceFoundWithHeaders( - "https://example.com/test.html", 2, false, requestHeadersOrder1); - org.zaproxy.zap.spider.parser.SpiderResourceFound spiderResourceFound2 = - createGetSpiderResourceFoundWithHeaders( - "https://example.com/test.html", 2, false, requestHeadersOrder2); - // When - spiderController.resourceFound(spiderResourceFound1); - spiderController.resourceFound(spiderResourceFound2); - // Then - verify(spider).submitTask(any()); - } - - @Test - void shouldNotSubmitSameGetTaskWithDifferentHeaderWhitespaces() { - // Given - List requestHeadersWithoutWS = new ArrayList<>(); - requestHeadersWithoutWS.add(new HttpHeaderField("X-Custom-Header-1", "xyz")); - List requestHeadersWithWS = new ArrayList<>(); - requestHeadersWithWS.add(new HttpHeaderField("\tX-Custom-Header-1 ", "\nxyz ")); - org.zaproxy.zap.spider.parser.SpiderResourceFound spiderResourceFound1 = - createGetSpiderResourceFoundWithHeaders( - "https://example.com/test.html", 2, false, requestHeadersWithoutWS); - org.zaproxy.zap.spider.parser.SpiderResourceFound spiderResourceFound2 = - createGetSpiderResourceFoundWithHeaders( - "https://example.com/test.html", 2, false, requestHeadersWithWS); - // When - spiderController.resourceFound(spiderResourceFound1); - spiderController.resourceFound(spiderResourceFound2); - // Then - verify(spider).submitTask(any()); - } - - @Test - void shouldNotSubmitSameGetTaskWithDifferentHeaderCases() { - // Given - List requestHeadersUpperCase = new ArrayList<>(); - requestHeadersUpperCase.add(new HttpHeaderField("X-CUSTOM-HEADER-1", "XYZ")); - List requestHeadersLowerCase = new ArrayList<>(); - requestHeadersLowerCase.add(new HttpHeaderField("x-custom-header-1", "xyz")); - org.zaproxy.zap.spider.parser.SpiderResourceFound spiderResourceFound1 = - createGetSpiderResourceFoundWithHeaders( - "https://example.com/test.html", 2, false, requestHeadersUpperCase); - org.zaproxy.zap.spider.parser.SpiderResourceFound spiderResourceFound2 = - createGetSpiderResourceFoundWithHeaders( - "https://example.com/test.html", 2, false, requestHeadersLowerCase); - // When - spiderController.resourceFound(spiderResourceFound1); - spiderController.resourceFound(spiderResourceFound2); - // Then - verify(spider).submitTask(any()); - } - - @Test - void shouldNotSubmitSameGetTaskWithDuplicateHeaders() { - // Given - List requestHeadersWithoutDuplicates = new ArrayList<>(); - requestHeadersWithoutDuplicates.add(new HttpHeaderField("X-Custom-Header-1", "xyz")); - List requestHeadersWithDuplicates = new ArrayList<>(); - requestHeadersWithDuplicates.add(new HttpHeaderField("X-Custom-Header-1", "xyz")); - requestHeadersWithDuplicates.add(new HttpHeaderField("X-Custom-Header-1", "xyz")); - requestHeadersWithDuplicates.add(new HttpHeaderField("X-Custom-Header-1", "xyz")); - org.zaproxy.zap.spider.parser.SpiderResourceFound spiderResourceFound1 = - createGetSpiderResourceFoundWithHeaders( - "https://example.com/test.html", 2, false, requestHeadersWithoutDuplicates); - org.zaproxy.zap.spider.parser.SpiderResourceFound spiderResourceFound2 = - createGetSpiderResourceFoundWithHeaders( - "https://example.com/test.html", 2, false, requestHeadersWithDuplicates); - // When - spiderController.resourceFound(spiderResourceFound1); - spiderController.resourceFound(spiderResourceFound2); - // Then - verify(spider).submitTask(any()); - } - - private static org.zaproxy.zap.spider.parser.SpiderResourceFound - createBasicGetSpiderResourceFound(String uri, int depth) { - return org.zaproxy.zap.spider.parser.SpiderResourceFound.builder() - .setDepth(depth) - .setUri(uri) - .build(); - } - - private static org.zaproxy.zap.spider.parser.SpiderResourceFound - createGetSpiderResourceFoundWithHeaders( - String uri, - int depth, - boolean shouldIgnore, - List requestHeaders) { - return org.zaproxy.zap.spider.parser.SpiderResourceFound.builder() - .setDepth(depth) - .setUri(uri) - .setShouldIgnore(shouldIgnore) - .setHeaders(requestHeaders) - .build(); - } - - private static org.zaproxy.zap.spider.parser.SpiderResourceFound - createBasicPostSpiderResourceFound(String uri, String body, int depth) { - return org.zaproxy.zap.spider.parser.SpiderResourceFound.builder() - .setDepth(depth) - .setUri(uri) - .setMethod(HttpRequestHeader.POST) - .setBody(body) - .build(); - } - - private static org.zaproxy.zap.spider.parser.SpiderResourceFound - createPostSpiderResourceFoundWithHeaders( - String uri, - String body, - int depth, - boolean shouldIgnore, - List requestHeaders) { - return org.zaproxy.zap.spider.parser.SpiderResourceFound.builder() - .setDepth(depth) - .setUri(uri) - .setShouldIgnore(shouldIgnore) - .setMethod(HttpRequestHeader.POST) - .setBody(body) - .setHeaders(requestHeaders) - .build(); - } -} diff --git a/zap/src/test/java/org/zaproxy/zap/spider/URLCanonicalizerUnitTest.java b/zap/src/test/java/org/zaproxy/zap/spider/URLCanonicalizerUnitTest.java deleted file mode 100644 index 69471febee0..00000000000 --- a/zap/src/test/java/org/zaproxy/zap/spider/URLCanonicalizerUnitTest.java +++ /dev/null @@ -1,624 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2013 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.spider; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; - -import java.util.Arrays; -import java.util.function.Predicate; -import org.apache.commons.httpclient.URI; -import org.apache.commons.httpclient.URIException; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; - -/** - * This test ensure that nothing was broken in the handling of normal URLs during the implementation - * of OData support as well as ensure that the OData support is correct. - * - *

It checks the canonicalization mechanism used to verify if a URL has already been visited - * before during the spider phase. - */ -@SuppressWarnings("deprecation") -class URLCanonicalizerUnitTest { - - @Test - void shouldRemoveDefaultPortOfHttpUriWhenCanonicalizing() { - // Given - String uri = "http://example.com:80/"; - // When - String canonicalizedUri = org.zaproxy.zap.spider.URLCanonicalizer.getCanonicalURL(uri); - // Then - assertThat(canonicalizedUri, is(equalTo("http://example.com/"))); - } - - @Test - void shouldNotRemoveNonDefaultPortOfHttpUriWhenCanonicalizing() { - // Given - String uri = "http://example.com:443/"; - // When - String canonicalizedUri = org.zaproxy.zap.spider.URLCanonicalizer.getCanonicalURL(uri); - // Then - assertThat(canonicalizedUri, is(equalTo("http://example.com:443/"))); - } - - @Test - void shouldRemoveDefaultPortOfHttpsUriWhenCanonicalizing() { - // Given - String uri = "https://example.com:443/"; - // When - String canonicalizedUri = org.zaproxy.zap.spider.URLCanonicalizer.getCanonicalURL(uri); - // Then - assertThat(canonicalizedUri, is(equalTo("https://example.com/"))); - } - - @Test - void shouldNotRemoveNonDefaultPortOfHttpsUriWhenCanonicalizing() { - // Given - String uri = "https://example.com:80/"; - // When - String canonicalizedUri = org.zaproxy.zap.spider.URLCanonicalizer.getCanonicalURL(uri); - // Then - assertThat(canonicalizedUri, is(equalTo("https://example.com:80/"))); - } - - @Test - void shouldAddEmptyPathIfUriHasNoPathWhenCanonicalizing() { - // Given - String uri = "http://example.com"; - // When - String canonicalizedUri = org.zaproxy.zap.spider.URLCanonicalizer.getCanonicalURL(uri); - // Then - assertThat(canonicalizedUri, is(equalTo("http://example.com/"))); - } - - @Test - void shouldCanonicalizeURIsWithAuthority() { - // Given - String[] uris = {"http://example.com/", "https://example.com/", "ftp://example.com/"}; - for (String uri : uris) { - // When - String canonicalizedUri = org.zaproxy.zap.spider.URLCanonicalizer.getCanonicalURL(uri); - // Then - assertThat(canonicalizedUri, canonicalizedUri, is(equalTo(uri))); - } - } - - @Test - void shouldUseBaseURIToResolveRelativeURIsWhenCanonicalizing() { - // Given - String baseURI = "http://example.com/path/"; - String[] relativeURIs = {"relative", "a/b/c", "../", "/absolute/path", ""}; - String[] expectedCanonicalURIs = { - "http://example.com/path/relative", - "http://example.com/path/a/b/c", - "http://example.com/", - "http://example.com/absolute/path", - "http://example.com/path/", - }; - for (int i = 0; i < relativeURIs.length; i++) { - // When - String canonicalizedUri = - org.zaproxy.zap.spider.URLCanonicalizer.getCanonicalURL( - relativeURIs[i], baseURI); - // Then - assertThat(canonicalizedUri, canonicalizedUri, is(equalTo(expectedCanonicalURIs[i]))); - } - } - - @Test - void shouldNormaliseEmptyAndDotPathSegmentsWhenCanonicalizing() { - // Given - String[] uris = { - "http://example.com/../../x", - "http://example.com/a//b/c//", - "http://example.com/a/./b/./c", - "http://example.com/.." - }; - String[] expectedCanonicalURIs = { - "http://example.com/x", - "http://example.com/a/b/c/", - "http://example.com/a/b/c", - "http://example.com/.." - }; - for (int i = 0; i < uris.length; i++) { - // When - String canonicalizedUri = - org.zaproxy.zap.spider.URLCanonicalizer.getCanonicalURL(uris[i]); - // Then - assertThat(canonicalizedUri, canonicalizedUri, is(equalTo(expectedCanonicalURIs[i]))); - } - } - - @ParameterizedTest - @ValueSource( - strings = { - "javascript:", - "javascript:ignore()", - "mailto:ignore@example.com", - "tel:+1-900-555-0191" - }) - void shouldIgnoreURIsWithNoAuthority(String uri) { - // Given / When - String canonicalizedUri = org.zaproxy.zap.spider.URLCanonicalizer.getCanonicalURL(uri); - // Then - assertThat(canonicalizedUri, canonicalizedUri, is(equalTo(null))); - } - - @Test - void shouldReturnCanonicalUriWithPercentEncodedPath() throws URIException { - // Given - String uri = new URI("http://example.com/path/%C3%A1/", true).toString(); - // When - String canonicalizedUri = org.zaproxy.zap.spider.URLCanonicalizer.getCanonicalURL(uri); - // Then - assertThat(canonicalizedUri, is(equalTo("http://example.com/path/%C3%A1/"))); - } - - @Test - void shouldReturnCanonicalUriWithPercentEncodedQuery() throws URIException { - // Given - String uri = new URI("http://example.com/path/?par%C3%A2m=v%C3%A3lue", true).toString(); - // When - String canonicalizedUri = org.zaproxy.zap.spider.URLCanonicalizer.getCanonicalURL(uri); - // Then - assertThat(canonicalizedUri, is(equalTo("http://example.com/path/?par%C3%A2m=v%C3%A3lue"))); - } - - @Test - void shouldCorrectlyParseQueryParameterNamesAndValuesWithAmpersandsAndEqualsWhenCanonicalizing() - throws URIException { - // Given - String uri = new URI("http://example.com/?par%26am%3D1=val%26u%3De1", true).toString(); - // When - String canonicalizedUri = org.zaproxy.zap.spider.URLCanonicalizer.getCanonicalURL(uri); - // Then - assertThat(canonicalizedUri, is(equalTo("http://example.com/?par%26am%3D1=val%26u%3De1"))); - } - - @Test - void shouldPreserveQueryParametersWithSameNameWhenCanonicalizing() throws URIException { - // Given - String uri = - new URI( - "http://example.com/?name1=value1.1&name1=value1.2&name2=value2&name2=value3", - true) - .toString(); - // When - String canonicalizedUri = org.zaproxy.zap.spider.URLCanonicalizer.getCanonicalURL(uri); - // Then - assertThat( - canonicalizedUri, - is( - equalTo( - "http://example.com/?name1=value1.1&name1=value1.2&name2=value2&name2=value3"))); - } - - @Test - void shouldSortQueryParametersByNameAndValueWhenCanonicalizing() throws URIException { - // Given - String uri = - new URI( - "http://example.com/?&name2=value2&name3=value3&name1=value1.2&name1=value1.1", - true) - .toString(); - // When - String canonicalizedUri = org.zaproxy.zap.spider.URLCanonicalizer.getCanonicalURL(uri); - // Then - assertThat( - canonicalizedUri, - is( - equalTo( - "http://example.com/?name1=value1.1&name1=value1.2&name2=value2&name3=value3"))); - } - - @Test - void shouldReturnPercentEncodedUriWhenCleaningParametersIn_USE_ALL_mode() throws URIException { - // Given - URI uri = new URI("http://example.com/path/%C3%A1/?par%C3%A2m=v%C3%A3lue", true); - // When - String cleanedUri = - org.zaproxy.zap.spider.URLCanonicalizer.buildCleanedParametersURIRepresentation( - uri, - org.zaproxy.zap.spider.SpiderParam.HandleParametersOption.USE_ALL, - false); - // Then - assertThat( - cleanedUri, is(equalTo("http://example.com/path/%C3%A1/?par%C3%A2m=v%C3%A3lue"))); - } - - @Test - void shouldReturnPercentEncodedUriWhenCleaningParametersIn_IGNORE_VALUE_mode() - throws URIException { - // Given - URI uri = new URI("http://example.com/path/%C3%A1/?par%C3%A2m=v%C3%A3lue1", true); - // When - String cleanedUri = - org.zaproxy.zap.spider.URLCanonicalizer.buildCleanedParametersURIRepresentation( - uri, - org.zaproxy.zap.spider.SpiderParam.HandleParametersOption.IGNORE_VALUE, - false); - // Then - assertThat(cleanedUri, is(equalTo("http://example.com/path/%C3%A1/?par%C3%A2m"))); - } - - @Test - void shouldReturnPercentEncodedUriWhenCleaningParametersIn_IGNORE_COMPLETELY_mode() - throws URIException { - // Given - URI uri = new URI("http://example.com/path/%C3%A1/?par%C3%A2m=v%C3%A3lue", true); - // When - String cleanedUri = - org.zaproxy.zap.spider.URLCanonicalizer.buildCleanedParametersURIRepresentation( - uri, - org.zaproxy.zap.spider.SpiderParam.HandleParametersOption.IGNORE_COMPLETELY, - false); - // Then - assertThat(cleanedUri, is(equalTo("http://example.com/path/%C3%A1/"))); - } - - @Test - void - shouldCorrectlyParseQueryParamNamesAndValuesWithAmpersandsAndEqualsWhenCleaningParametersIn_USE_ALL_mode() - throws URIException { - // Given - URI uri = new URI("http://example.com/path/?par%3Dam1=val%26ue1&par%26am2=val%3Due2", true); - // When - String cleanedUri = - org.zaproxy.zap.spider.URLCanonicalizer.buildCleanedParametersURIRepresentation( - uri, - org.zaproxy.zap.spider.SpiderParam.HandleParametersOption.USE_ALL, - false); - // Then - assertThat( - cleanedUri, - is(equalTo("http://example.com/path/?par%3Dam1=val%26ue1&par%26am2=val%3Due2"))); - } - - @Test - void - shouldCorrectlyParseQueryParamNamesAndValuesWithAmpersandsAndEqualsWhenCleaningParametersIn_IGNORE_VALUE_mode() - throws URIException { - // Given - URI uri = new URI("http://example.com/path/?par%3Dam1=val%26ue1&par%26am2=val%3Due2", true); - // When - String cleanedUri = - org.zaproxy.zap.spider.URLCanonicalizer.buildCleanedParametersURIRepresentation( - uri, - org.zaproxy.zap.spider.SpiderParam.HandleParametersOption.IGNORE_VALUE, - false); - // Then - assertThat(cleanedUri, is(equalTo("http://example.com/path/?par%26am2&par%3Dam1"))); - } - - @Test - void - shouldCorrectlyParseQueryParamNamesAndValuesWithAmpersandsAndEqualsWhenCleaningParametersIn_IGNORE_COMPLETELY_mode() - throws URIException { - // Given - URI uri = new URI("http://example.com/path/?par%3Dam1=val%26ue1&par%26am2=val%3Due2", true); - // When - String cleanedUri = - org.zaproxy.zap.spider.URLCanonicalizer.buildCleanedParametersURIRepresentation( - uri, - org.zaproxy.zap.spider.SpiderParam.HandleParametersOption.IGNORE_COMPLETELY, - false); - // Then - assertThat(cleanedUri, is(equalTo("http://example.com/path/"))); - } - - @Test - void shouldPreserveQueryParametersWithSameNameWhenCleaningParametersIn_USE_ALL_mode() - throws URIException { - // Given - URI uri = - new URI( - "http://example.com/path/?param%5B%5D=value1.1¶m%5B%5D=value1.2¶m2=value2", - true); - // When - String cleanedUri = - org.zaproxy.zap.spider.URLCanonicalizer.buildCleanedParametersURIRepresentation( - uri, - org.zaproxy.zap.spider.SpiderParam.HandleParametersOption.USE_ALL, - false); - // Then - assertThat( - cleanedUri, - is( - equalTo( - "http://example.com/path/?param%5B%5D=value1.1¶m%5B%5D=value1.2¶m2=value2"))); - } - - @Test - void shouldKeepJustOneQueryParameterWithSameNameWhenCleaningParametersIn_IGNORE_VALUE_mode() - throws URIException { - // Given - URI uri = - new URI( - "http://example.com/path/?param1=value1.1¶m1=value1.2¶m2=value2", - true); - // When - String cleanedUri = - org.zaproxy.zap.spider.URLCanonicalizer.buildCleanedParametersURIRepresentation( - uri, - org.zaproxy.zap.spider.SpiderParam.HandleParametersOption.IGNORE_VALUE, - false); - // Then - assertThat(cleanedUri, is(equalTo("http://example.com/path/?param1¶m2"))); - } - - @Test - void shouldRemoveAllQueryParametersWhenCleaningParametersIn_IGNORE_COMPLETELY_mode() - throws URIException { - // Given - URI uri = - new URI( - "http://example.com/path/?param1=value1.1¶m1=value1.2¶m2=value2", - true); - // When - String cleanedUri = - org.zaproxy.zap.spider.URLCanonicalizer.buildCleanedParametersURIRepresentation( - uri, - org.zaproxy.zap.spider.SpiderParam.HandleParametersOption.IGNORE_COMPLETELY, - false); - // Then - assertThat(cleanedUri, is(equalTo("http://example.com/path/"))); - } - - // Test of the legacy behavior - - @Test - void shouldCanonicalizeNormalURLWithoutParametersIn_USE_ALL_mode() throws URIException { - URI uri = new URI("http", null, "host", 9001, "/myservlet"); - String visitedURI = - org.zaproxy.zap.spider.URLCanonicalizer.buildCleanedParametersURIRepresentation( - uri, - org.zaproxy.zap.spider.SpiderParam.HandleParametersOption.USE_ALL, - false /* handleODataParametersVisited */); - assertThat(visitedURI, is("http://host:9001/myservlet")); - } - - @Test - void shouldCanonicalizeNormalURLWithoutParametersIn_IGNORE_COMPLETELY_mode() - throws URIException { - URI uri = new URI("http", null, "host", 9001, "/myservlet"); - String visitedURI = - org.zaproxy.zap.spider.URLCanonicalizer.buildCleanedParametersURIRepresentation( - uri, - org.zaproxy.zap.spider.SpiderParam.HandleParametersOption.IGNORE_COMPLETELY, - false /* handleODataParametersVisited */); - assertThat(visitedURI, is("http://host:9001/myservlet")); - } - - @Test - void shouldCanonicalizeNormalURLWithoutParametersIn_IGNORE_VALUE_mode() throws URIException { - URI uri = new URI("http", null, "host", 9001, "/myservlet"); - String visitedURI = - org.zaproxy.zap.spider.URLCanonicalizer.buildCleanedParametersURIRepresentation( - uri, - org.zaproxy.zap.spider.SpiderParam.HandleParametersOption.IGNORE_VALUE, - false /* handleODataParametersVisited */); - assertThat(visitedURI, is("http://host:9001/myservlet")); - } - - @Test - void shouldCanonicalizeNormalURLWithParametersIn_USE_ALL_mode() throws URIException { - - URI uri = new URI("http", null, "host", 9001, "/myservlet", "p1=2&p2=myparam"); - String visitedURI = - org.zaproxy.zap.spider.URLCanonicalizer.buildCleanedParametersURIRepresentation( - uri, - org.zaproxy.zap.spider.SpiderParam.HandleParametersOption.USE_ALL, - false /* handleODataParametersVisited */); - assertThat(visitedURI, is("http://host:9001/myservlet?p1=2&p2=myparam")); - } - - @Test - void shouldCanonicalizeNormalURLWithParametersIn_IGNORE_COMPLETELY_mode() throws URIException { - URI uri = new URI("http", null, "host", 9001, "/myservlet", "p1=2&p2=myparam"); - String visitedURI = - org.zaproxy.zap.spider.URLCanonicalizer.buildCleanedParametersURIRepresentation( - uri, - org.zaproxy.zap.spider.SpiderParam.HandleParametersOption.IGNORE_COMPLETELY, - false /* handleODataParametersVisited */); - assertThat(visitedURI, is("http://host:9001/myservlet")); - } - - @Test - void shouldCanonicalizeNormalURLWithParametersIn_IGNORE_VALUE_mode() throws URIException { - URI uri = new URI("http", null, "host", 9001, "/myservlet", "p1=2&p2=myparam"); - String visitedURI = - org.zaproxy.zap.spider.URLCanonicalizer.buildCleanedParametersURIRepresentation( - uri, - org.zaproxy.zap.spider.SpiderParam.HandleParametersOption.IGNORE_VALUE, - false /* handleODataParametersVisited */); - assertThat(visitedURI, is("http://host:9001/myservlet?p1&p2")); - } - - // Test the OData behavior - - @Test - void shouldCanonicalizeODataIDSimpleIn_USE_ALL_mode() throws URIException { - org.zaproxy.zap.spider.SpiderParam.HandleParametersOption spiderOption = - org.zaproxy.zap.spider.SpiderParam.HandleParametersOption.USE_ALL; - - URI uri = new URI("http", null, "host", 9001, "/app.svc/Book(1)"); - String visitedURI = - org.zaproxy.zap.spider.URLCanonicalizer.buildCleanedParametersURIRepresentation( - uri, spiderOption, true /* handleODataParametersVisited */); - assertThat(visitedURI, is("http://host:9001/app.svc/Book(1)")); - - uri = new URI("http", null, "host", 9001, "/app.svc/Book(1)/Author"); - visitedURI = - org.zaproxy.zap.spider.URLCanonicalizer.buildCleanedParametersURIRepresentation( - uri, spiderOption, true /* handleODataParametersVisited */); - assertThat(visitedURI, is("http://host:9001/app.svc/Book(1)/Author")); - } - - @Test - void shouldCanonicalizeODataIDSimpleIn_IGNORE_COMPLETELY_mode() throws URIException { - org.zaproxy.zap.spider.SpiderParam.HandleParametersOption spiderOption = - org.zaproxy.zap.spider.SpiderParam.HandleParametersOption.IGNORE_COMPLETELY; - - URI uri = new URI("http", null, "host", 9001, "/app.svc/Book(1)"); - String visitedURI = - org.zaproxy.zap.spider.URLCanonicalizer.buildCleanedParametersURIRepresentation( - uri, spiderOption, true /* handleODataParametersVisited */); - assertThat(visitedURI, is("http://host:9001/app.svc/Book()")); - - uri = new URI("http", null, "host", 9001, "/app.svc/Book(1)/Author"); - visitedURI = - org.zaproxy.zap.spider.URLCanonicalizer.buildCleanedParametersURIRepresentation( - uri, spiderOption, true /* handleODataParametersVisited */); - assertThat(visitedURI, is("http://host:9001/app.svc/Book()/Author")); - } - - @Test - void shouldCanonicalizeODataIDSimpleIn_IGNORE_VALUE_mode() throws URIException { - org.zaproxy.zap.spider.SpiderParam.HandleParametersOption spiderOption = - org.zaproxy.zap.spider.SpiderParam.HandleParametersOption.IGNORE_VALUE; - - URI uri = new URI("http", null, "host", 9001, "/app.svc/Book(1)"); - String visitedURI = - org.zaproxy.zap.spider.URLCanonicalizer.buildCleanedParametersURIRepresentation( - uri, spiderOption, true /* handleODataParametersVisited */); - assertThat(visitedURI, is("http://host:9001/app.svc/Book()")); - - uri = new URI("http", null, "host", 9001, "/app.svc/Book(1)/Author"); - visitedURI = - org.zaproxy.zap.spider.URLCanonicalizer.buildCleanedParametersURIRepresentation( - uri, spiderOption, true /* handleODataParametersVisited */); - assertThat(visitedURI, is("http://host:9001/app.svc/Book()/Author")); - } - - @Test - void shouldCanonicalizeODataIDMultipleIn_USE_ALL_mode() throws URIException { - org.zaproxy.zap.spider.SpiderParam.HandleParametersOption spiderOption = - org.zaproxy.zap.spider.SpiderParam.HandleParametersOption.USE_ALL; - - URI uri = new URI("http", null, "host", 9001, "/app.svc/Book(title='dummy',year=2012)"); - String visitedURI = - org.zaproxy.zap.spider.URLCanonicalizer.buildCleanedParametersURIRepresentation( - uri, spiderOption, true /* handleODataParametersVisited */); - assertThat(visitedURI, is("http://host:9001/app.svc/Book(title='dummy',year=2012)")); - - uri = new URI("http", null, "host", 9001, "/app.svc/Book(title='dummy',year=2012)/Author"); - visitedURI = - org.zaproxy.zap.spider.URLCanonicalizer.buildCleanedParametersURIRepresentation( - uri, spiderOption, true /* handleODataParametersVisited */); - assertThat(visitedURI, is("http://host:9001/app.svc/Book(title='dummy',year=2012)/Author")); - } - - @Test - void shouldCanonicalizeODataIDMultipleIn_IGNORE_COMPLETELY_mode() throws URIException { - org.zaproxy.zap.spider.SpiderParam.HandleParametersOption spiderOption = - org.zaproxy.zap.spider.SpiderParam.HandleParametersOption.IGNORE_COMPLETELY; - - URI uri = new URI("http", null, "host", 9001, "/app.svc/Book(title='dummy',year=2012)"); - String visitedURI = - org.zaproxy.zap.spider.URLCanonicalizer.buildCleanedParametersURIRepresentation( - uri, spiderOption, true /* handleODataParametersVisited */); - assertThat(visitedURI, is("http://host:9001/app.svc/Book()")); - - uri = new URI("http", null, "host", 9001, "/app.svc/Book(title='dummy',year=2012)/Author"); - visitedURI = - org.zaproxy.zap.spider.URLCanonicalizer.buildCleanedParametersURIRepresentation( - uri, spiderOption, true /* handleODataParametersVisited */); - assertThat(visitedURI, is("http://host:9001/app.svc/Book()/Author")); - } - - @Test - void shouldCanonicalizeODataIDMultipleIn_IGNORE_VALUE_mode() throws URIException { - org.zaproxy.zap.spider.SpiderParam.HandleParametersOption spiderOption = - org.zaproxy.zap.spider.SpiderParam.HandleParametersOption.IGNORE_VALUE; - - URI uri = new URI("http", null, "host", 9001, "/app.svc/Book(title='dummy',year=2012)"); - String visitedURI = - org.zaproxy.zap.spider.URLCanonicalizer.buildCleanedParametersURIRepresentation( - uri, spiderOption, true /* handleODataParametersVisited */); - assertThat(visitedURI, is("http://host:9001/app.svc/Book(title,year)")); - - uri = new URI("http", null, "host", 9001, "/app.svc/Book(title='dummy',year=2012)/Author"); - visitedURI = - org.zaproxy.zap.spider.URLCanonicalizer.buildCleanedParametersURIRepresentation( - uri, spiderOption, true /* handleODataParametersVisited */); - assertThat(visitedURI, is("http://host:9001/app.svc/Book(title,year)/Author")); - } - - @Test - void shouldSkipIrrelevantQueryParametersWhenCanonicalizing() throws URIException { - // Given - String uri = - new URI("http://example.com/?name1=value1&name2=value2&name3=value3", true) - .toString(); - - Predicate irrelevantParameters = Arrays.asList("name1", "name3")::contains; - // When - String canonicalizedUri = URLCanonicalizer.getCanonicalURL(uri, null, irrelevantParameters); - // Then - assertThat(canonicalizedUri, is(equalTo("http://example.com/?name2=value2"))); - } - - @Test - void shouldSkipSessionTokensWhenCanonicalizing() throws URIException { - // Given - String uri = - new URI("http://example.com/?jsessionid=id1&phpsessid=id2&aspsessionid=id3", true) - .toString(); - // When - String canonicalizedUri = URLCanonicalizer.getCanonicalURL(uri, null); - // Then - assertThat(canonicalizedUri, is(equalTo("http://example.com/"))); - } - - @Test - void shouldSkipIrrelevantQueryParametersWhenCleaningParametersIn_IGNORE_VALUE_mode() - throws URIException { - // Given - URI uri = new URI("http://example.com/?name1=value1&name2=value2&name3=value3", true); - Predicate irrelevantParameters = Arrays.asList("name1", "name3")::contains; - // When - String cleanedUri = - URLCanonicalizer.buildCleanedParametersURIRepresentation( - uri, - org.zaproxy.zap.spider.SpiderParam.HandleParametersOption.IGNORE_VALUE, - false, - irrelevantParameters); - // Then - assertThat(cleanedUri, is(equalTo("http://example.com/?name2"))); - } - - @Test - void shouldSkipSessionTokensWhenCleaningParametersIn_IGNORE_VALUE_mode() throws URIException { - // Given - URI uri = - new URI("http://example.com/?jsessionid=id1&phpsessid=id2&aspsessionid=id3", true); - // When - String cleanedUri = - URLCanonicalizer.buildCleanedParametersURIRepresentation( - uri, - org.zaproxy.zap.spider.SpiderParam.HandleParametersOption.IGNORE_VALUE, - false); - // Then - assertThat(cleanedUri, is(equalTo("http://example.com/"))); - } -} diff --git a/zap/src/test/java/org/zaproxy/zap/spider/URLResolverRfc1808ExamplesUnitTest.java b/zap/src/test/java/org/zaproxy/zap/spider/URLResolverRfc1808ExamplesUnitTest.java deleted file mode 100644 index a7affd13886..00000000000 --- a/zap/src/test/java/org/zaproxy/zap/spider/URLResolverRfc1808ExamplesUnitTest.java +++ /dev/null @@ -1,154 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2012 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.spider; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.is; - -import org.junit.jupiter.api.Test; - -/** Unit test for RFC 1808 compliance of {@link org.zaproxy.zap.spider.URLResolver}. */ -@SuppressWarnings("deprecation") -class URLResolverRfc1808ExamplesUnitTest { - - /** - * @see RFC 1808 - 5.1. Normal - * Examples - */ - @Test - void resolveRfc1808NormalExamples() { - final String baseUrl = "http://a/b/c/d;p?q#f"; - - assertThat(org.zaproxy.zap.spider.URLResolver.resolveUrl(baseUrl, "g:h"), is("g:h")); - assertThat( - org.zaproxy.zap.spider.URLResolver.resolveUrl(baseUrl, "g"), is("http://a/b/c/g")); - assertThat( - org.zaproxy.zap.spider.URLResolver.resolveUrl(baseUrl, "./g"), - is("http://a/b/c/g")); - assertThat( - org.zaproxy.zap.spider.URLResolver.resolveUrl(baseUrl, "g/"), - is("http://a/b/c/g/")); - assertThat(org.zaproxy.zap.spider.URLResolver.resolveUrl(baseUrl, "/g"), is("http://a/g")); - assertThat(org.zaproxy.zap.spider.URLResolver.resolveUrl(baseUrl, "//g"), is("http://g")); - assertThat( - org.zaproxy.zap.spider.URLResolver.resolveUrl(baseUrl, "?y"), - is("http://a/b/c/d;p?y")); - assertThat( - org.zaproxy.zap.spider.URLResolver.resolveUrl(baseUrl, "g?y"), - is("http://a/b/c/g?y")); - assertThat( - org.zaproxy.zap.spider.URLResolver.resolveUrl(baseUrl, "g?y/./x"), - is("http://a/b/c/g?y/./x")); - assertThat( - org.zaproxy.zap.spider.URLResolver.resolveUrl(baseUrl, "#s"), - is("http://a/b/c/d;p?q#s")); - assertThat( - org.zaproxy.zap.spider.URLResolver.resolveUrl(baseUrl, "g#s"), - is("http://a/b/c/g#s")); - assertThat( - org.zaproxy.zap.spider.URLResolver.resolveUrl(baseUrl, "g#s/./x"), - is("http://a/b/c/g#s/./x")); - assertThat( - org.zaproxy.zap.spider.URLResolver.resolveUrl(baseUrl, "g?y#s"), - is("http://a/b/c/g?y#s")); - assertThat( - org.zaproxy.zap.spider.URLResolver.resolveUrl(baseUrl, ";x"), - is("http://a/b/c/d;x")); - assertThat( - org.zaproxy.zap.spider.URLResolver.resolveUrl(baseUrl, "g;x"), - is("http://a/b/c/g;x")); - assertThat( - org.zaproxy.zap.spider.URLResolver.resolveUrl(baseUrl, "g;x?y#s"), - is("http://a/b/c/g;x?y#s")); - assertThat( - org.zaproxy.zap.spider.URLResolver.resolveUrl(baseUrl, "."), is("http://a/b/c/")); - assertThat( - org.zaproxy.zap.spider.URLResolver.resolveUrl(baseUrl, "./"), is("http://a/b/c/")); - assertThat(org.zaproxy.zap.spider.URLResolver.resolveUrl(baseUrl, ".."), is("http://a/b/")); - assertThat( - org.zaproxy.zap.spider.URLResolver.resolveUrl(baseUrl, "../"), is("http://a/b/")); - assertThat( - org.zaproxy.zap.spider.URLResolver.resolveUrl(baseUrl, "../g"), is("http://a/b/g")); - assertThat( - org.zaproxy.zap.spider.URLResolver.resolveUrl(baseUrl, "../.."), is("http://a/")); - assertThat( - org.zaproxy.zap.spider.URLResolver.resolveUrl(baseUrl, "../../"), is("http://a/")); - assertThat( - org.zaproxy.zap.spider.URLResolver.resolveUrl(baseUrl, "../../g"), - is("http://a/g")); - } - - /** - * @see RFC 1808 - 5.2. Abnormal - * Examples - */ - @Test - void resolveRfc1808AbnormalExamples() { - final String baseUrl = "http://a/b/c/d;p?q#f"; - - assertThat( - org.zaproxy.zap.spider.URLResolver.resolveUrl(baseUrl, ""), - is("http://a/b/c/d;p?q#f")); - - // Deviations from RFC 1808 to match browsers' behaviour. - // Expected by RFC 1808: - // assertThat(URLResolver.resolveUrl(baseUrl, "../../../g"), is("http://a/../g")); - // assertThat(URLResolver.resolveUrl(baseUrl, "../../../../g"), is("http://a/../../g")); - // assertThat(URLResolver.resolveUrl(baseUrl, "/../g"), is("http://a/../g")); - // Browsers' behaviour: - assertThat( - org.zaproxy.zap.spider.URLResolver.resolveUrl(baseUrl, "../../../g"), - is("http://a/g")); - assertThat( - org.zaproxy.zap.spider.URLResolver.resolveUrl(baseUrl, "../../../../g"), - is("http://a/g")); - assertThat( - org.zaproxy.zap.spider.URLResolver.resolveUrl(baseUrl, "/../g"), is("http://a/g")); - - assertThat( - org.zaproxy.zap.spider.URLResolver.resolveUrl(baseUrl, "/./g"), is("http://a/./g")); - assertThat( - org.zaproxy.zap.spider.URLResolver.resolveUrl(baseUrl, "g."), - is("http://a/b/c/g.")); - assertThat( - org.zaproxy.zap.spider.URLResolver.resolveUrl(baseUrl, ".g"), - is("http://a/b/c/.g")); - assertThat( - org.zaproxy.zap.spider.URLResolver.resolveUrl(baseUrl, "g.."), - is("http://a/b/c/g..")); - assertThat( - org.zaproxy.zap.spider.URLResolver.resolveUrl(baseUrl, "..g"), - is("http://a/b/c/..g")); - assertThat( - org.zaproxy.zap.spider.URLResolver.resolveUrl(baseUrl, "./../g"), - is("http://a/b/g")); - assertThat( - org.zaproxy.zap.spider.URLResolver.resolveUrl(baseUrl, "./g/."), - is("http://a/b/c/g/")); - assertThat( - org.zaproxy.zap.spider.URLResolver.resolveUrl(baseUrl, "g/./h"), - is("http://a/b/c/g/h")); - assertThat( - org.zaproxy.zap.spider.URLResolver.resolveUrl(baseUrl, "g/../h"), - is("http://a/b/c/h")); - assertThat(org.zaproxy.zap.spider.URLResolver.resolveUrl(baseUrl, "http:g"), is("http:g")); - assertThat(org.zaproxy.zap.spider.URLResolver.resolveUrl(baseUrl, "http:"), is("http:")); - } -} diff --git a/zap/src/test/java/org/zaproxy/zap/spider/URLResolverUnitTest.java b/zap/src/test/java/org/zaproxy/zap/spider/URLResolverUnitTest.java deleted file mode 100644 index e364ca33330..00000000000 --- a/zap/src/test/java/org/zaproxy/zap/spider/URLResolverUnitTest.java +++ /dev/null @@ -1,171 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2012 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.spider; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.is; -import static org.junit.jupiter.api.Assertions.assertThrows; - -import org.junit.jupiter.api.Test; - -/** - * Unit test for {@link org.zaproxy.zap.spider.URLResolver}. - * - * @author bjoern.kimminich@gmx.de - */ -@SuppressWarnings("deprecation") -class URLResolverUnitTest { - - @Test - void shouldThrowExceptionOnMissingBaseUrl() { - // Given - String baseUrl = null; - // When / Then - assertThrows( - IllegalArgumentException.class, - () -> org.zaproxy.zap.spider.URLResolver.resolveUrl(baseUrl, "notNull")); - } - - @Test - void shouldThrowExceptionOnMissingRelativeUrl() { - // Given - String relativeUrl = null; - // When / Then - assertThrows( - IllegalArgumentException.class, - () -> org.zaproxy.zap.spider.URLResolver.resolveUrl("notNull", relativeUrl)); - } - - @Test - void shouldAppendRelativeUrlToBaseUrlHost() { - assertThat( - org.zaproxy.zap.spider.URLResolver.resolveUrl("http://www.abc.de", "/xy/z"), - is("http://www.abc.de/xy/z")); - } - - @Test - void shouldInsertSlashBetweenBaseUrlAndRelativeUrlIfMissing() { - assertThat( - org.zaproxy.zap.spider.URLResolver.resolveUrl("http://www.abc.de", "xyz"), - is("http://www.abc.de/xyz")); - } - - @Test - void shouldReplaceLastPartOfUrlPathFromBaseUrlWithRelativeUrl() { - assertThat( - org.zaproxy.zap.spider.URLResolver.resolveUrl("http://www.abc.de/w/x", "y/z"), - is("http://www.abc.de/w/y/z")); - } - - @Test - void shouldRemoveFragmentFromBaseUrlBeforeAppendingRelativeUrl() { - assertThat( - org.zaproxy.zap.spider.URLResolver.resolveUrl("http://www.abc.de#anchor", "y"), - is("http://www.abc.de/y")); - } - - @Test - void shouldRemoveQueryFromBaseUrlBeforeAppendingRelativeUrl() { - assertThat( - org.zaproxy.zap.spider.URLResolver.resolveUrl("http://www.abc.de?y=z", "test"), - is("http://www.abc.de/test")); - } - - @Test - void shouldRemoveParametersFromBaseUrlBeforeAppendingRelativeUrl() { - assertThat( - org.zaproxy.zap.spider.URLResolver.resolveUrl("http://www.abc.de;y;z", "test"), - is("http://www.abc.de/test")); - } - - @Test - void shouldReturnOriginalBaseUrlForGivenEmptyRelativeUrl() { - assertThat( - org.zaproxy.zap.spider.URLResolver.resolveUrl( - "http://www.abc.de/x?y=z&u=v#123", ""), - is("http://www.abc.de/x?y=z&u=v#123")); - } - - @Test - void shouldReturnOriginalRelativeUrlForGivenAbsoluteUrlAsRelativeUrl() { - assertThat( - org.zaproxy.zap.spider.URLResolver.resolveUrl( - "http://base.url", "http://www.abc.de/x?y=z&u=v#123"), - is("http://www.abc.de/x?y=z&u=v#123")); - } - - @Test - void shouldUseSchemeOfBaseUrlForGivenUrlWithHostAsRelativeUrl() { - assertThat( - org.zaproxy.zap.spider.URLResolver.resolveUrl("https://base.url", "//www.test.com"), - is("https://www.test.com")); - } - - @Test - void shouldAppendQueryGivenAsRelativeUrlToBaseUrl() { - assertThat( - org.zaproxy.zap.spider.URLResolver.resolveUrl("http://abc.de/123", "?x=y"), - is("http://abc.de/123?x=y")); - } - - @Test - void shouldAppendParametersGivenAsRelativeUrlToBaseUrl() { - assertThat( - org.zaproxy.zap.spider.URLResolver.resolveUrl("http://abc.de/123", ";x=y"), - is("http://abc.de/123;x=y")); - } - - @Test - void shouldAppendFragmentGivenAsRelativeUrlToBaseUrl() { - assertThat( - org.zaproxy.zap.spider.URLResolver.resolveUrl("http://abc.de/123", "#test"), - is("http://abc.de/123#test")); - } - - @Test - void shouldRemoveLeadingSlashPointsFromRelativeUrlBeforeAppendingToBaseUrl() { - assertThat( - org.zaproxy.zap.spider.URLResolver.resolveUrl("http://abc.de/123/xyz", "../test"), - is("http://abc.de/test")); - } - - @Test - void shouldRemoveAllSlashPointSlashOccurrencesFromResolvedUrl() { - assertThat( - org.zaproxy.zap.spider.URLResolver.resolveUrl( - "http://abc.de/./", "test/./xyz/./123"), - is("http://abc.de/test/xyz/123")); - } - - @Test - void shouldRemoveTrailingPointFromResolvedUrl() { - assertThat( - org.zaproxy.zap.spider.URLResolver.resolveUrl("http://abc.de", "test/."), - is("http://abc.de/test/")); - } - - @Test - void shouldApplyDirectoryTraversalWithSlashPointsInResolvedUrl() { - assertThat( - org.zaproxy.zap.spider.URLResolver.resolveUrl( - "http://abc.de/x/../", "y/../z/../test/123/.."), - is("http://abc.de/test/")); - } -} diff --git a/zap/src/test/java/org/zaproxy/zap/spider/filters/DefaultFetchFilterUnitTest.java b/zap/src/test/java/org/zaproxy/zap/spider/filters/DefaultFetchFilterUnitTest.java deleted file mode 100644 index f7a513f086c..00000000000 --- a/zap/src/test/java/org/zaproxy/zap/spider/filters/DefaultFetchFilterUnitTest.java +++ /dev/null @@ -1,301 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2017 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.spider.filters; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.BDDMockito.given; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import org.apache.commons.httpclient.URI; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.zaproxy.zap.model.Context; - -/** Unit test for {@link DefaultFetchFilter}. */ -@SuppressWarnings("deprecation") -@ExtendWith(MockitoExtension.class) -class DefaultFetchFilterUnitTest { - - @Mock Context context; - - private org.zaproxy.zap.spider.filters.DefaultFetchFilter filter; - - @BeforeEach - void setUp() { - filter = new org.zaproxy.zap.spider.filters.DefaultFetchFilter(); - } - - @Test - void shouldFilterUriWithNonSchemeAsIllegalProtocol() throws Exception { - // Given - URI uri = createUri("example.com"); - // When - org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus status = filter.checkFilter(uri); - // Then - assertThat( - status, - is( - equalTo( - org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus - .ILLEGAL_PROTOCOL))); - } - - @Test - void shouldFilterUriWithNonHttpOrHttpsSchemeAsIllegalProtocol() throws Exception { - // Given - URI uri = createUri("ftp://example.com"); - // When - org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus status = filter.checkFilter(uri); - // Then - assertThat( - status, - is( - equalTo( - org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus - .ILLEGAL_PROTOCOL))); - } - - @Test - void shouldFilterUriWithHttpSchemeAsOutOfScopeByDefault() throws Exception { - // Given - URI uri = createUri("http://example.com"); - // When - org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus status = filter.checkFilter(uri); - // Then - assertThat( - status, - is(equalTo(org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus.OUT_OF_SCOPE))); - } - - @Test - void shouldFilterUriWithHttpsSchemeAsOutOfScopeByDefault() throws Exception { - // Given - URI uri = createUri("https://example.com"); - // When - org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus status = filter.checkFilter(uri); - // Then - assertThat( - status, - is(equalTo(org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus.OUT_OF_SCOPE))); - } - - @Test - void shouldFilterOutOfScopeUriAsOutOfScope() throws Exception { - // Given - filter.addScopeRegex("scope.example.com"); - URI uri = createUri("http://example.com"); - // When - org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus status = filter.checkFilter(uri); - // Then - assertThat( - status, - is(equalTo(org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus.OUT_OF_SCOPE))); - } - - @Test - void shouldFilterInScopeUriAsValid() throws Exception { - // Given - filter.addScopeRegex("example.com"); - URI uri = createUri("http://example.com"); - // When - org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus status = filter.checkFilter(uri); - // Then - assertThat( - status, is(equalTo(org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus.VALID))); - } - - @Test - void shouldFilterNonAlwaysInScopeUriAsOutOfScope() throws Exception { - // Given - filter.setDomainsAlwaysInScope(domainsAlwaysInScope("scope.example.com")); - URI uri = createUri("https://example.com"); - // When - org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus status = filter.checkFilter(uri); - // Then - assertThat( - status, - is(equalTo(org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus.OUT_OF_SCOPE))); - } - - @Test - void shouldFilterAlwaysInScopeUriAsValid() throws Exception { - // Given - filter.setDomainsAlwaysInScope(domainsAlwaysInScope("example.com")); - URI uri = createUri("https://example.com"); - // When - org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus status = filter.checkFilter(uri); - // Then - assertThat( - status, is(equalTo(org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus.VALID))); - } - - @Test - void shouldFilterExcludedInScopeUriAsUserRules() throws Exception { - // Given - filter.addScopeRegex("example.com"); - filter.setExcludeRegexes(excludeRegexes(".*example\\.com.*")); - URI uri = createUri("http://example.com"); - // When - org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus status = filter.checkFilter(uri); - // Then - assertThat( - status, - is(equalTo(org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus.USER_RULES))); - } - - @Test - void shouldFilterExcludedAlwaysInScopeUriAsUserRules() throws Exception { - // Given - filter.setDomainsAlwaysInScope(domainsAlwaysInScope("example.com")); - filter.setExcludeRegexes(excludeRegexes(".*example\\.com.*")); - URI uri = createUri("http://example.com"); - // When - org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus status = filter.checkFilter(uri); - // Then - assertThat( - status, - is(equalTo(org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus.USER_RULES))); - } - - @Test - void shouldFilterNonExcludedInScopeUriAsValid() throws Exception { - // Given - filter.addScopeRegex("example.com"); - filter.setExcludeRegexes(excludeRegexes("subdomain\\.example\\.com.*")); - URI uri = createUri("http://example.com"); - // When - org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus status = filter.checkFilter(uri); - // Then - assertThat( - status, is(equalTo(org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus.VALID))); - } - - @Test - void shouldFilterNonExcludedAlwaysInScopeUriAsValid() throws Exception { - // Given - filter.setDomainsAlwaysInScope(domainsAlwaysInScope("example.com")); - filter.setExcludeRegexes(excludeRegexes("subdomain\\.example\\.com.*")); - URI uri = createUri("http://example.com"); - // When - org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus status = filter.checkFilter(uri); - // Then - assertThat( - status, is(equalTo(org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus.VALID))); - } - - @Test - void shouldFilterOutOfContextUriAsOutOfContext() throws Exception { - // Given - filter.setScanContext(contextInScope(false)); - URI uri = createUri("http://example.com"); - // When - org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus status = filter.checkFilter(uri); - // Then - assertThat( - status, - is(equalTo(org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus.OUT_OF_CONTEXT))); - } - - @Test - void shouldFilterInContextUriAsValid() throws Exception { - // Given - filter.setScanContext(contextInScope(true)); - URI uri = createUri("http://example.com"); - // When - org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus status = filter.checkFilter(uri); - // Then - assertThat( - status, is(equalTo(org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus.VALID))); - } - - @Test - void shouldFilterExcludedInContextUriAsUserRules() throws Exception { - // Given - filter.setScanContext(contextInScope(true)); - filter.setExcludeRegexes(excludeRegexes(".*example\\.com.*")); - URI uri = createUri("http://example.com"); - // When - org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus status = filter.checkFilter(uri); - // Then - assertThat( - status, - is(equalTo(org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus.USER_RULES))); - } - - @Test - void shouldFilterNonExcludedInContextUriAsValid() throws Exception { - // Given - filter.setScanContext(contextInScope(true)); - filter.setExcludeRegexes(excludeRegexes("subdomain\\.example\\.com.*")); - URI uri = createUri("http://example.com"); - // When - org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus status = filter.checkFilter(uri); - // Then - assertThat( - status, is(equalTo(org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus.VALID))); - } - - private static URI createUri(String uri) { - try { - return new URI(uri, true); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - private static List domainsAlwaysInScope( - String... domains) { - if (domains == null || domains.length == 0) { - return Collections.emptyList(); - } - - List domainsAlwaysInScope = - new ArrayList<>(1); - for (String domain : domains) { - domainsAlwaysInScope.add(new org.zaproxy.zap.spider.DomainAlwaysInScopeMatcher(domain)); - } - return domainsAlwaysInScope; - } - - private static List excludeRegexes(String... regexes) { - if (regexes == null || regexes.length == 0) { - return Collections.emptyList(); - } - - List excludedRegexes = new ArrayList<>(1); - for (String regex : regexes) { - excludedRegexes.add(regex); - } - return excludedRegexes; - } - - private Context contextInScope(boolean inScope) { - given(context.isInContext(anyString())).willReturn(inScope); - return context; - } -} diff --git a/zap/src/test/java/org/zaproxy/zap/spider/filters/DefaultParseFilterUnitTest.java b/zap/src/test/java/org/zaproxy/zap/spider/filters/DefaultParseFilterUnitTest.java deleted file mode 100644 index d8a7f51920c..00000000000 --- a/zap/src/test/java/org/zaproxy/zap/spider/filters/DefaultParseFilterUnitTest.java +++ /dev/null @@ -1,373 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2017 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.spider.filters; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; -import static org.junit.jupiter.api.Assertions.assertThrows; - -import java.util.Collections; -import java.util.Enumeration; -import java.util.ResourceBundle; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.parosproxy.paros.network.HttpHeader; -import org.parosproxy.paros.network.HttpMessage; -import org.parosproxy.paros.network.HttpRequestHeader; - -/** Unit test for {@link DefaultParseFilter}. */ -@SuppressWarnings("deprecation") -class DefaultParseFilterUnitTest { - - private static final String FILTERED_REASON_EMPTY = "empty"; - private static final String FILTERED_REASON_MAX_SIZE = "maxSize"; - private static final String FILTERED_REASON_NOT_TEXT = "notText"; - - private ResourceBundle resourceBundle; - - @BeforeEach - void setUp() throws Exception { - resourceBundle = - new ResourceBundle() { - - @Override - protected Object handleGetObject(String key) { - switch (key) { - case "spider.parsefilter.reason.empty": - return FILTERED_REASON_EMPTY; - case "spider.parsefilter.reason.maxsize": - return FILTERED_REASON_MAX_SIZE; - case "spider.parsefilter.reason.nottext": - return FILTERED_REASON_NOT_TEXT; - } - return null; - } - - @Override - public Enumeration getKeys() { - return Collections.emptyEnumeration(); - } - }; - } - - @Test - @SuppressWarnings("deprecation") - void shouldCreateDefaultParseFilterWithDefaultConfigsAndResourceBundleIfNoneSet() { - assertDoesNotThrow(() -> new DefaultParseFilter()); - } - - @Test - void shouldCreateDefaultParseFilterWithConfigsAndResourceBundleSet() { - // Given - org.zaproxy.zap.spider.SpiderParam configs = new org.zaproxy.zap.spider.SpiderParam(); - // When / Then - assertDoesNotThrow(() -> new DefaultParseFilter(configs, resourceBundle)); - } - - @Test - void shouldFailToCreateDefaultParseFilterWithNullConfigs() { - // Given - org.zaproxy.zap.spider.SpiderParam configs = null; - // When / Then - assertThrows( - IllegalArgumentException.class, - () -> new DefaultParseFilter(configs, resourceBundle)); - } - - @Test - void shouldFailToCreateDefaultParseFilterWithNullResourceBundle() { - // Given - ResourceBundle resourceBundle = null; - org.zaproxy.zap.spider.SpiderParam configs = new org.zaproxy.zap.spider.SpiderParam(); - // When / Then - assertThrows( - IllegalArgumentException.class, - () -> new DefaultParseFilter(configs, resourceBundle)); - } - - @Test - void shouldFilterNullHttpMessage() { - // Given - DefaultParseFilter filter = createDefaultParseFilter(); - HttpMessage httpMessage = null; - // When - org.zaproxy.zap.spider.filters.ParseFilter.FilterResult filterResult = - filter.filtered(httpMessage); - // Then - assertThat(filterResult.isFiltered(), is(equalTo(true))); - assertThat(filterResult.getReason(), is(equalTo(FILTERED_REASON_EMPTY))); - } - - @Test - void shouldFilterHttpMessageWithEmptyRequestHeader() { - // Given - DefaultParseFilter filter = createDefaultParseFilter(); - HttpMessage httpMessage = new HttpMessage(); - // When - org.zaproxy.zap.spider.filters.ParseFilter.FilterResult filterResult = - filter.filtered(httpMessage); - // Then - assertThat(filterResult.isFiltered(), is(equalTo(true))); - assertThat(filterResult.getReason(), is(equalTo(FILTERED_REASON_EMPTY))); - } - - @Test - void shouldFilterHttpMessageWithEmptyResponseHeader() { - // Given - DefaultParseFilter filter = createDefaultParseFilter(); - HttpMessage httpMessage = createDefaultRequest(); - // When - org.zaproxy.zap.spider.filters.ParseFilter.FilterResult filterResult = - filter.filtered(httpMessage); - // Then - assertThat(filterResult.isFiltered(), is(equalTo(true))); - assertThat(filterResult.getReason(), is(equalTo(FILTERED_REASON_EMPTY))); - } - - @Test - void shouldHandleHttpMessageWithNoPathInRequestUri() throws Exception { - // Given - DefaultParseFilter filter = createDefaultParseFilter(); - HttpMessage httpMessage = createHttpMessageWithRequestUri("http://example.com"); - // When / Then - assertDoesNotThrow(() -> filter.filtered(httpMessage)); - } - - @Test - void shouldNotFilterHttpMessageWithSvnXmlRequest() throws Exception { - // Given - DefaultParseFilter filter = createDefaultParseFilter(); - HttpMessage httpMessage = createHttpMessageWithRequestUri("/.svn/wc.db"); - // When - org.zaproxy.zap.spider.filters.ParseFilter.FilterResult filterResult = - filter.filtered(httpMessage); - // Then - assertThat(filterResult.isFiltered(), is(equalTo(false))); - } - - @Test - void shouldNotFilterHttpMessageWithSvnDbRequest() throws Exception { - // Given - DefaultParseFilter filter = createDefaultParseFilter(); - HttpMessage httpMessage = createHttpMessageWithRequestUri("/.svn/entries"); - // When - org.zaproxy.zap.spider.filters.ParseFilter.FilterResult filterResult = - filter.filtered(httpMessage); - // Then - assertThat(filterResult.isFiltered(), is(equalTo(false))); - } - - @Test - void shouldNotFilterHttpMessageWithGitRequest() throws Exception { - // Given - DefaultParseFilter filter = createDefaultParseFilter(); - HttpMessage httpMessage = createHttpMessageWithRequestUri("/.git/index"); - // When - org.zaproxy.zap.spider.filters.ParseFilter.FilterResult filterResult = - filter.filtered(httpMessage); - // Then - assertThat(filterResult.isFiltered(), is(equalTo(false))); - } - - @Test - void shouldNotFilterHttpMessageWithRobotsTxtRequestEvenWithoutContentType() throws Exception { - // Given - DefaultParseFilter filter = createDefaultParseFilter(); - HttpMessage httpMessage = createHttpMessageWithRequestUri("/robots.txt"); - httpMessage.getResponseHeader().setHeader(HttpHeader.CONTENT_TYPE, ""); - // When - org.zaproxy.zap.spider.filters.ParseFilter.FilterResult filterResult = - filter.filtered(httpMessage); - // Then - assertThat(filterResult.isFiltered(), is(equalTo(false))); - } - - @Test - void shouldNotFilterHttpMessageWithRobotsTxtRequestEvenWithContentType() throws Exception { - // Given - DefaultParseFilter filter = createDefaultParseFilter(); - HttpMessage httpMessage = createHttpMessageWithRequestUri("/robots.txt"); - httpMessage.getResponseHeader().setHeader(HttpHeader.CONTENT_TYPE, "text/plain"); - // When - org.zaproxy.zap.spider.filters.ParseFilter.FilterResult filterResult = - filter.filtered(httpMessage); - // Then - assertThat(filterResult.isFiltered(), is(equalTo(false))); - } - - @Test - void shouldNotFilterHttpMessageWithSitemapXmlRequestEvenWithoutContentType() throws Exception { - // Given - DefaultParseFilter filter = createDefaultParseFilter(); - HttpMessage httpMessage = createHttpMessageWithRequestUri("/sitemap.xml"); - httpMessage.getResponseHeader().setHeader(HttpHeader.CONTENT_TYPE, ""); - // When - org.zaproxy.zap.spider.filters.ParseFilter.FilterResult filterResult = - filter.filtered(httpMessage); - // Then - assertThat(filterResult.isFiltered(), is(equalTo(false))); - } - - @Test - void shouldNotFilterHttpMessageWithSitemapXmlRequestEvenWithContentType() throws Exception { - // Given - DefaultParseFilter filter = createDefaultParseFilter(); - HttpMessage httpMessage = createHttpMessageWithRequestUri("/sitemap.xml"); - httpMessage.getResponseHeader().setHeader(HttpHeader.CONTENT_TYPE, "application/xml"); - // When - org.zaproxy.zap.spider.filters.ParseFilter.FilterResult filterResult = - filter.filtered(httpMessage); - // Then - assertThat(filterResult.isFiltered(), is(equalTo(false))); - } - - @Test - void shouldFilterHttpMessageWithNonTextResponse() throws Exception { - // Given - DefaultParseFilter filter = createDefaultParseFilter(); - HttpMessage httpMessage = createDefaultRequest(); - httpMessage.setResponseHeader("HTTP/1.1 200 OK\r\nContent-Type: application/x-binary\r\n"); - // When - org.zaproxy.zap.spider.filters.ParseFilter.FilterResult filterResult = - filter.filtered(httpMessage); - // Then - assertThat(filterResult.isFiltered(), is(equalTo(true))); - assertThat(filterResult.getReason(), is(equalTo(FILTERED_REASON_NOT_TEXT))); - } - - @Test - void shouldNotFilterHttpMessageWithTextResponse() throws Exception { - // Given - DefaultParseFilter filter = createDefaultParseFilter(); - HttpMessage httpMessage = createDefaultRequest(); - httpMessage.setResponseHeader("HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n"); - // When - org.zaproxy.zap.spider.filters.ParseFilter.FilterResult filterResult = - filter.filtered(httpMessage); - // Then - assertThat(filterResult.isFiltered(), is(equalTo(false))); - } - - @Test - void shouldNotFilterHttpMessageWithRedirectResponse() throws Exception { - // Given - DefaultParseFilter filter = createDefaultParseFilter(); - HttpMessage httpMessage = createDefaultRequest(); - httpMessage.setResponseHeader("HTTP/1.1 303 See Other\r\n"); - // When - org.zaproxy.zap.spider.filters.ParseFilter.FilterResult filterResult = - filter.filtered(httpMessage); - // Then - assertThat(filterResult.isFiltered(), is(equalTo(false))); - } - - @Test - void shouldFilterHttpMessageWithResponseAboveMaxParseSize() throws Exception { - // Given - int maxParseSizeBytes = 2; - DefaultParseFilter filter = - new DefaultParseFilter(createSpiderParam(maxParseSizeBytes), resourceBundle); - HttpMessage httpMessage = createHttpMessageWithResponseBody("ABC"); - // When - org.zaproxy.zap.spider.filters.ParseFilter.FilterResult filterResult = - filter.filtered(httpMessage); - // Then - assertThat(filterResult.isFiltered(), is(equalTo(true))); - assertThat(filterResult.getReason(), is(equalTo(FILTERED_REASON_MAX_SIZE))); - } - - @Test - void shouldNotFilterHttpMessageWithResponseEqualToMaxParseSize() throws Exception { - // Given - int maxParseSizeBytes = 2; - DefaultParseFilter filter = - new DefaultParseFilter(createSpiderParam(maxParseSizeBytes), resourceBundle); - HttpMessage httpMessage = createHttpMessageWithResponseBody("AB"); - // When - org.zaproxy.zap.spider.filters.ParseFilter.FilterResult filterResult = - filter.filtered(httpMessage); - // Then - assertThat(filterResult.isFiltered(), is(equalTo(false))); - } - - @Test - void shouldNotFilterHttpMessageWithResponseUnderMaxParseSize() throws Exception { - // Given - int maxParseSizeBytes = 2; - DefaultParseFilter filter = - new DefaultParseFilter(createSpiderParam(maxParseSizeBytes), resourceBundle); - HttpMessage httpMessage = createHttpMessageWithResponseBody("A"); - // When - org.zaproxy.zap.spider.filters.ParseFilter.FilterResult filterResult = - filter.filtered(httpMessage); - // Then - assertThat(filterResult.isFiltered(), is(equalTo(false))); - } - - private DefaultParseFilter createDefaultParseFilter() { - return new DefaultParseFilter(createSpiderParam(Integer.MAX_VALUE), resourceBundle); - } - - private static org.zaproxy.zap.spider.SpiderParam createSpiderParam( - final int maxParseSizeBytes) { - return new org.zaproxy.zap.spider.SpiderParam() { - - @Override - public int getMaxParseSizeBytes() { - return maxParseSizeBytes; - } - }; - } - - private static HttpMessage createHttpMessageWithResponseBody(String responseBody) { - try { - HttpMessage message = createDefaultRequest(); - message.setResponseHeader("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\n"); - message.setResponseBody(responseBody); - return message; - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - private static HttpMessage createDefaultRequest() { - try { - return new HttpMessage( - new HttpRequestHeader("GET / HTTP/1.1\r\nHost: example.com\r\n")); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - private static HttpMessage createHttpMessageWithRequestUri(String requestUri) { - try { - HttpMessage message = - new HttpMessage( - new HttpRequestHeader( - "GET " + requestUri + " HTTP/1.1\r\nHost: example.com\r\n")); - message.setResponseHeader("HTTP/1.1 200 OK\r\n"); - return message; - } catch (Exception e) { - throw new RuntimeException(e); - } - } -} diff --git a/zap/src/test/java/org/zaproxy/zap/spider/filters/HttpPrefixFetchFilterUnitTest.java b/zap/src/test/java/org/zaproxy/zap/spider/filters/HttpPrefixFetchFilterUnitTest.java deleted file mode 100644 index 6f882061308..00000000000 --- a/zap/src/test/java/org/zaproxy/zap/spider/filters/HttpPrefixFetchFilterUnitTest.java +++ /dev/null @@ -1,418 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2016 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.spider.filters; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; -import static org.junit.jupiter.api.Assertions.assertThrows; - -import org.apache.commons.httpclient.URI; -import org.junit.jupiter.api.Test; - -/** Unit test for {@link HttpPrefixFetchFilter}. */ -@SuppressWarnings("deprecation") -class HttpPrefixFetchFilterUnitTest { - - @Test - void shouldFailToCreateFetchFilterWithUndefinedURI() { - // Given - URI undefinedUri = null; - // When / Then - assertThrows( - IllegalArgumentException.class, - () -> new org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter(undefinedUri)); - } - - @Test - void shouldFailToCreateFetchFilterWithNoScheme() throws Exception { - // Given - URI prefixUri = new URI("example.org/", true); - // When / Then - assertThrows( - IllegalArgumentException.class, - () -> new org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter(prefixUri)); - } - - @Test - void shouldFailToCreateFetchFilterWithNonHttpOrHttpsScheme() throws Exception { - // Given - URI prefixUri = new URI("ftp://example.org/", true); - // When / Then - assertThrows( - IllegalArgumentException.class, - () -> new org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter(prefixUri)); - } - - @Test - void shouldFailToCreateFetchFilterWithNoHost() throws Exception { - // Given - URI prefixUri = new URI("http://", true); - // When / Then - assertThrows( - IllegalArgumentException.class, - () -> new org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter(prefixUri)); - } - - @Test - void shouldFailToCreateFetchFilterWithMalformedHost() throws Exception { - // Given - URI prefixUri = new URI("http://a%0/", true); - // When / Then - assertThrows( - IllegalArgumentException.class, - () -> new org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter(prefixUri)); - } - - @Test - void shouldNotAddPathToNormalisedPrefixIfPrefixDoesNotHavePath() throws Exception { - // Given - URI prefixUri = new URI("http://example.org", true); - org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter fetchFilter = - new org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter(prefixUri); - // When - String normalisedPrefix = fetchFilter.getNormalisedPrefix(); - // Then - assertThat(normalisedPrefix, is(equalTo("http://example.org"))); - } - - @Test - void shouldDiscardUserInfoFromPrefix() throws Exception { - // Given - URI prefixUri = new URI("http://user:pass@example.org", true); - org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter fetchFilter = - new org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter(prefixUri); - // When - String normalisedPrefix = fetchFilter.getNormalisedPrefix(); - // Then - assertThat(normalisedPrefix, is(equalTo("http://example.org"))); - } - - @Test - void shouldDiscardEverythingAfterPathComponentFromPrefix() throws Exception { - // Given - URI prefixUri = new URI("https://example.org/path?query#fragment", true); - org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter fetchFilter = - new org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter(prefixUri); - // When - String normalisedPrefix = fetchFilter.getNormalisedPrefix(); - // Then - assertThat(normalisedPrefix, is(equalTo("https://example.org/path"))); - } - - @Test - void shouldDiscardDefaultHttpPortFromPrefix() throws Exception { - // Given - URI prefixUri = new URI("http://example.org:80/", true); - org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter fetchFilter = - new org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter(prefixUri); - // When - String normalisedPrefix = fetchFilter.getNormalisedPrefix(); - // Then - assertThat(normalisedPrefix, is(equalTo("http://example.org/"))); - } - - @Test - void shouldDiscardDefaultHttpsPortFromPrefix() throws Exception { - // Given - URI prefixUri = new URI("https://example.org:443/", true); - org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter fetchFilter = - new org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter(prefixUri); - // When - String normalisedPrefix = fetchFilter.getNormalisedPrefix(); - // Then - assertThat(normalisedPrefix, is(equalTo("https://example.org/"))); - } - - @Test - void shouldKeepNonDefaultPortFromPrefix() throws Exception { - // Given - URI prefixUri = new URI("https://example.org:8443/", true); - org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter fetchFilter = - new org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter(prefixUri); - // When - String normalisedPrefix = fetchFilter.getNormalisedPrefix(); - // Then - assertThat(normalisedPrefix, is(equalTo("https://example.org:8443/"))); - } - - @Test - void shouldKeepDefaultHttpPortInHttpsPrefix() throws Exception { - // Given - URI prefixUri = new URI("https://example.org:80/", true); - org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter fetchFilter = - new org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter(prefixUri); - // When - String normalisedPrefix = fetchFilter.getNormalisedPrefix(); - // Then - assertThat(normalisedPrefix, is(equalTo("https://example.org:80/"))); - } - - @Test - void shouldKeepDefaultHttpsPortInHttpPrefix() throws Exception { - // Given - URI prefixUri = new URI("http://example.org:443/", true); - org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter fetchFilter = - new org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter(prefixUri); - // When - String normalisedPrefix = fetchFilter.getNormalisedPrefix(); - // Then - assertThat(normalisedPrefix, is(equalTo("http://example.org:443/"))); - } - - @Test - void shouldFilterUndefinedUriAsOutOfScope() throws Exception { - // Given - URI prefixUri = new URI("http://example.org/", true); - org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter fetchFilter = - new org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter(prefixUri); - // When - org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus filterStatus = - fetchFilter.checkFilter(null); - // Then - assertThat( - filterStatus, - is(equalTo(org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus.OUT_OF_SCOPE))); - } - - @Test - void shouldFilterUriWithNoSchemeAsOutOfScope() throws Exception { - // Given - URI prefixUri = new URI("http://example.org/", true); - org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter fetchFilter = - new org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter(prefixUri); - URI uri = new URI("/path", true); - // When - org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus filterStatus = - fetchFilter.checkFilter(uri); - // Then - assertThat( - filterStatus, - is(equalTo(org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus.OUT_OF_SCOPE))); - } - - @Test - void shouldFilterUriWithNonHttpOrHttpsSchemeAsOutOfScope() throws Exception { - // Given - URI prefixUri = new URI("http://example.org/", true); - org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter fetchFilter = - new org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter(prefixUri); - URI uri = new URI("ftp://example.org/", true); - // When - org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus filterStatus = - fetchFilter.checkFilter(uri); - // Then - assertThat( - filterStatus, - is(equalTo(org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus.OUT_OF_SCOPE))); - } - - @Test - void shouldFilterUriWithNoHostAsOutOfScope() throws Exception { - // Given - URI prefixUri = new URI("http://example.org/", true); - org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter fetchFilter = - new org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter(prefixUri); - URI uri = new URI("http://", true); - // When - org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus filterStatus = - fetchFilter.checkFilter(uri); - // Then - assertThat( - filterStatus, - is(equalTo(org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus.OUT_OF_SCOPE))); - } - - @Test - void shouldFilterUriWithMalformedHostAsOutOfScope() throws Exception { - // Given - URI prefixUri = new URI("http://example.org/", true); - org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter fetchFilter = - new org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter(prefixUri); - URI uri = new URI("http://a%0/", true); - // When - org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus filterStatus = - fetchFilter.checkFilter(uri); - // Then - assertThat( - filterStatus, - is(equalTo(org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus.OUT_OF_SCOPE))); - } - - @Test - void shouldFilterUriWithDifferentSchemeAsOutOfScope() throws Exception { - // Given - URI prefixUri = new URI("http://example.org/", true); - org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter fetchFilter = - new org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter(prefixUri); - URI uri = new URI("https://example.org/", true); - // When - org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus filterStatus = - fetchFilter.checkFilter(uri); - // Then - assertThat( - filterStatus, - is(equalTo(org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus.OUT_OF_SCOPE))); - } - - @Test - void shouldFilterUriWithDifferentSchemeButSamePortAsOutOfScope() throws Exception { - // Given - URI prefixUri = new URI("http://example.org/", true); - org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter fetchFilter = - new org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter(prefixUri); - URI uri = new URI("https://example.org:80/", true); - // When - org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus filterStatus = - fetchFilter.checkFilter(uri); - // Then - assertThat( - filterStatus, - is(equalTo(org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus.OUT_OF_SCOPE))); - } - - @Test - void shouldFilterUriWithDifferentPortAsOutOfScope() throws Exception { - // Given - URI prefixUri = new URI("http://example.org/", true); - org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter fetchFilter = - new org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter(prefixUri); - URI uri = new URI("http://example.org:1234/", true); - // When - org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus filterStatus = - fetchFilter.checkFilter(uri); - // Then - assertThat( - filterStatus, - is(equalTo(org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus.OUT_OF_SCOPE))); - } - - @Test - void shouldFilterUriWithDifferentHostAsOutOfScope() throws Exception { - // Given - URI prefixUri = new URI("http://example.org/", true); - org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter fetchFilter = - new org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter(prefixUri); - URI uri = new URI("http://domain.example.org/", true); - // When - org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus filterStatus = - fetchFilter.checkFilter(uri); - // Then - assertThat( - filterStatus, - is(equalTo(org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus.OUT_OF_SCOPE))); - } - - @Test - void shouldFilterUriWithDifferentSmallerPathAsOutOfScope() throws Exception { - // Given - URI prefixUri = new URI("http://example.org/path", true); - org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter fetchFilter = - new org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter(prefixUri); - URI uri = new URI("http://example.org/p", true); - // When - org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus filterStatus = - fetchFilter.checkFilter(uri); - // Then - assertThat( - filterStatus, - is(equalTo(org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus.OUT_OF_SCOPE))); - } - - @Test - void shouldFilterUriWithDifferentPathAsOutOfScope() throws Exception { - // Given - URI prefixUri = new URI("http://example.org/path", true); - org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter fetchFilter = - new org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter(prefixUri); - URI uri = new URI("http://example.org/not/same/path", true); - // When - org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus filterStatus = - fetchFilter.checkFilter(uri); - // Then - assertThat( - filterStatus, - is(equalTo(org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus.OUT_OF_SCOPE))); - } - - @Test - void shouldFilterUriWithDifferentNonEmptyPathAsOutOfScope() throws Exception { - // Given - URI prefixUri = new URI("http://example.org/", true); - org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter fetchFilter = - new org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter(prefixUri); - URI uri = new URI("http://example.org", true); - // When - org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus filterStatus = - fetchFilter.checkFilter(uri); - // Then - assertThat( - filterStatus, - is(equalTo(org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus.OUT_OF_SCOPE))); - } - - @Test - void shouldFilterUriWithSamePathPrefixAsValid() throws Exception { - // Given - URI prefixUri = new URI("http://example.org/path", true); - org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter fetchFilter = - new org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter(prefixUri); - URI uri = new URI("http://example.org/path/subtree", true); - // When - org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus filterStatus = - fetchFilter.checkFilter(uri); - // Then - assertThat( - filterStatus, - is(equalTo(org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus.VALID))); - } - - @Test - void shouldFilterUriAsValidWhenPathPrefixIsEmpty() throws Exception { - // Given - URI prefixUri = new URI("http://example.org", true); - org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter fetchFilter = - new org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter(prefixUri); - URI uri = new URI("http://example.org/path/subtree", true); - // When - org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus filterStatus = - fetchFilter.checkFilter(uri); - // Then - assertThat( - filterStatus, - is(equalTo(org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus.VALID))); - } - - @Test - void shouldFilterUriWithSamePathPrefixEvenIfHasQueryOrFragmentAsValid() throws Exception { - // Given - URI prefixUri = new URI("http://example.org/path", true); - org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter fetchFilter = - new org.zaproxy.zap.spider.filters.HttpPrefixFetchFilter(prefixUri); - URI uri = new URI("http://example.org/path/subtree/a?query#fragment", true); - // When - org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus filterStatus = - fetchFilter.checkFilter(uri); - // Then - assertThat( - filterStatus, - is(equalTo(org.zaproxy.zap.spider.filters.FetchFilter.FetchStatus.VALID))); - } -} diff --git a/zap/src/test/java/org/zaproxy/zap/spider/parser/SpiderHtmlFormParserUnitTest.java b/zap/src/test/java/org/zaproxy/zap/spider/parser/SpiderHtmlFormParserUnitTest.java deleted file mode 100644 index 14b6429b174..00000000000 --- a/zap/src/test/java/org/zaproxy/zap/spider/parser/SpiderHtmlFormParserUnitTest.java +++ /dev/null @@ -1,1858 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2016 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.spider.parser; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.contains; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasSize; -import static org.hamcrest.Matchers.is; -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; -import static org.junit.jupiter.api.Assertions.assertThrows; - -import java.nio.file.Path; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import net.htmlparser.jericho.Source; -import org.apache.commons.httpclient.URI; -import org.junit.jupiter.api.Test; -import org.parosproxy.paros.network.HttpMessage; -import org.zaproxy.zap.utils.Pair; - -/** Unit test for {@link SpiderHtmlFormParser}. */ -@SuppressWarnings({"deprecation", "removal"}) -class SpiderHtmlFormParserUnitTest extends SpiderParserTestUtils { - - private static final String FORM_METHOD_TOKEN = "%%METHOD%%"; - private static final String FORM_ACTION_TOKEN = "%%ACTION%%"; - private static final String BASE_HTML_TOKEN = "%%BASE_HTML%%"; - - private static final String ROOT_PATH = "/"; - private static final int BASE_DEPTH = 0; - - private static final Path BASE_DIR_HTML_FILES = - getResourcePath("htmlform", SpiderHtmlFormParserUnitTest.class); - - @Test - void shouldFailToCreateParserWithUndefinedSpiderOptions() { - // Given - org.zaproxy.zap.spider.SpiderParam undefinedSpiderOptions = null; - // When / Then - assertThrows( - NullPointerException.class, - () -> - new org.zaproxy.zap.spider.parser.SpiderHtmlFormParser( - undefinedSpiderOptions, - new org.zaproxy.zap.model.DefaultValueGenerator())); - } - - @Test - void shouldFailToEvaluateAnUndefinedMessage() { - // Given - HttpMessage undefinedMessage = null; - org.zaproxy.zap.spider.parser.SpiderHtmlFormParser htmlParser = - createSpiderHtmlFormParser(); - // When / Then - assertThrows( - NullPointerException.class, - () -> htmlParser.canParseResource(undefinedMessage, ROOT_PATH, false)); - } - - @Test - void shouldNotParseMessageIfAlreadyParsed() { - // Given - org.zaproxy.zap.spider.parser.SpiderHtmlFormParser htmlParser = - createSpiderHtmlFormParser(); - boolean parsed = true; - // When - boolean canParse = htmlParser.canParseResource(new HttpMessage(), ROOT_PATH, parsed); - // Then - assertThat(canParse, is(equalTo(false))); - } - - @Test - void shouldNotParseNonHtmlResponse() { - // Given - org.zaproxy.zap.spider.parser.SpiderHtmlFormParser htmlParser = - createSpiderHtmlFormParser(); - boolean parsed = false; - // When - boolean canParse = htmlParser.canParseResource(new HttpMessage(), ROOT_PATH, parsed); - // Then - assertThat(canParse, is(equalTo(false))); - } - - @Test - void shouldParseHtmlResponse() { - // Given - org.zaproxy.zap.spider.parser.SpiderHtmlFormParser htmlParser = - createSpiderHtmlFormParser(); - HttpMessage messageHtmlResponse = createMessageWith("NoForms.html"); - boolean parsed = false; - // When - boolean canParse = htmlParser.canParseResource(messageHtmlResponse, ROOT_PATH, parsed); - // Then - assertThat(canParse, is(equalTo(true))); - } - - @Test - void shouldParseHtmlResponseEvenIfProvidedPathIsNull() { - // Given - org.zaproxy.zap.spider.parser.SpiderHtmlFormParser htmlParser = - createSpiderHtmlFormParser(); - HttpMessage messageHtmlResponse = createMessageWith("NoForms.html"); - boolean parsed = false; - // When - boolean canParse = htmlParser.canParseResource(messageHtmlResponse, null, parsed); - // Then - assertThat(canParse, is(equalTo(true))); - } - - @Test - void shouldNotParseHtmlResponseIfAlreadyParsed() { - // Given - org.zaproxy.zap.spider.parser.SpiderHtmlFormParser htmlParser = - createSpiderHtmlFormParser(); - HttpMessage messageHtmlResponse = createMessageWith("NoForms.html"); - boolean parsed = true; - // When - boolean canParse = htmlParser.canParseResource(messageHtmlResponse, ROOT_PATH, parsed); - // Then - assertThat(canParse, is(equalTo(false))); - } - - @Test - void shouldFailToParseAnUndefinedMessage() { - // Given - HttpMessage undefinedMessage = null; - org.zaproxy.zap.spider.parser.SpiderHtmlFormParser htmlParser = - createSpiderHtmlFormParser(); - Source source = createSource(createMessageWith("NoForms.html")); - // When / Then - assertThrows( - NullPointerException.class, - () -> htmlParser.parseResource(undefinedMessage, source, BASE_DEPTH)); - } - - @Test - void shouldNotParseMessageIfFormProcessingIsDisabled() { - // Given - org.zaproxy.zap.spider.SpiderParam spiderOptions = createSpiderParamWithConfig(); - spiderOptions.setProcessForm(false); - org.zaproxy.zap.spider.parser.SpiderHtmlFormParser htmlParser = - new org.zaproxy.zap.spider.parser.SpiderHtmlFormParser( - spiderOptions, new org.zaproxy.zap.model.DefaultValueGenerator()); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage messageHtmlResponse = createMessageWith("PostGetForms.html"); - Source source = createSource(messageHtmlResponse); - // When - boolean completelyParsed = - htmlParser.parseResource(messageHtmlResponse, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(0))); - } - - @Test - void shouldParseMessageEvenWithoutSource() { - // Given - Source source = null; - org.zaproxy.zap.spider.parser.SpiderHtmlFormParser htmlParser = - createSpiderHtmlFormParser(); - HttpMessage messageHtmlResponse = createMessageWith("NoForms.html"); - // When / Then - assertDoesNotThrow(() -> htmlParser.parseResource(messageHtmlResponse, source, BASE_DEPTH)); - } - - @Test - void shouldNeverConsiderCompletelyParsed() { - // Given - org.zaproxy.zap.spider.parser.SpiderHtmlFormParser htmlParser = - createSpiderHtmlFormParser(); - HttpMessage messageHtmlResponse = createMessageWith("NoForms.html"); - Source source = createSource(messageHtmlResponse); - // When - boolean completelyParsed = - htmlParser.parseResource(messageHtmlResponse, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - } - - @Test - void shouldParseSingleGetForm() { - // Given - org.zaproxy.zap.spider.parser.SpiderHtmlFormParser htmlParser = - createSpiderHtmlFormParser(); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage messageHtmlResponse = createMessageWith("GET", "Form.html"); - Source source = createSource(messageHtmlResponse); - // When - boolean completelyParsed = - htmlParser.parseResource(messageHtmlResponse, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(1))); - assertThat( - listener.getUrlsFound(), - contains("http://example.org/?field1=Text+1&field2=Text+2&submit=Submit")); - } - - @Test - void shouldParseMultipleGetForms() { - // Given - org.zaproxy.zap.spider.parser.SpiderHtmlFormParser htmlParser = - createSpiderHtmlFormParser(); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage messageHtmlResponse = createMessageWith("GET", "Forms.html"); - Source source = createSource(messageHtmlResponse); - // When - boolean completelyParsed = - htmlParser.parseResource(messageHtmlResponse, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(2))); - assertThat( - listener.getUrlsFound(), - contains( - "http://example.org/form1?field1=Text+1&field2=Text+2&submit=Submit", - "http://example.org/form2?a=x&b=y&c=z")); - } - - @Test - void shouldParseGetFormWithMultipleSubmitFields() { - // Given - org.zaproxy.zap.spider.parser.SpiderHtmlFormParser htmlParser = - createSpiderHtmlFormParser(); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage messageHtmlResponse = createMessageWith("GET", "FormMultipleSubmitFields.html"); - Source source = createSource(messageHtmlResponse); - // When - boolean completelyParsed = - htmlParser.parseResource(messageHtmlResponse, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(5))); - assertThat( - listener.getUrlsFound(), - contains( - "http://example.org/?field1=Text+1&field2=Text+2&submit1=Submit+1", - "http://example.org/?field1=Text+1&field2=Text+2&submit2=Submit+2", - "http://example.org/?field1=Text+1&field2=Text+2&submit3=Submit+3", - "http://example.org/?field1=Text+1&field2=Text+2&submit=Submit+4", - "http://example.org/?field1=Text+1&field2=Text+2&submit=Submit+5")); - } - - @Test - void shouldParseSinglePostForm() { - // Given - org.zaproxy.zap.spider.parser.SpiderHtmlFormParser htmlParser = - createSpiderHtmlFormParser(); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage msg = createMessageWith("POST", "Form.html"); - Source source = createSource(msg); - // When - boolean completelyParsed = htmlParser.parseResource(msg, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(1))); - assertThat( - listener.getResourcesFound(), - contains( - postResource( - msg, - 1, - "http://example.org/", - "field1=Text+1&field2=Text+2&submit=Submit"))); - } - - @Test - void shouldParseMultiplePostForms() { - // Given - org.zaproxy.zap.spider.parser.SpiderHtmlFormParser htmlParser = - createSpiderHtmlFormParser(); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage msg = createMessageWith("POST", "Forms.html"); - Source source = createSource(msg); - // When - boolean completelyParsed = htmlParser.parseResource(msg, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(2))); - assertThat( - listener.getResourcesFound(), - contains( - postResource( - msg, - 1, - "http://example.org/form1", - "field1=Text+1&field2=Text+2&submit=Submit"), - postResource(msg, 1, "http://example.org/form2", "a=x&b=y&c=z"))); - } - - @Test - void shouldParsePostFormWithMultipleSubmitFields() { - // Given - org.zaproxy.zap.spider.parser.SpiderHtmlFormParser htmlParser = - createSpiderHtmlFormParser(); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage msg = createMessageWith("POST", "FormMultipleSubmitFields.html"); - Source source = createSource(msg); - // When - boolean completelyParsed = htmlParser.parseResource(msg, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(5))); - assertThat( - listener.getResourcesFound(), - contains( - postResource( - msg, - 1, - "http://example.org/", - "field1=Text+1&field2=Text+2&submit1=Submit+1"), - postResource( - msg, - 1, - "http://example.org/", - "field1=Text+1&field2=Text+2&submit2=Submit+2"), - postResource( - msg, - 1, - "http://example.org/", - "field1=Text+1&field2=Text+2&submit3=Submit+3"), - postResource( - msg, - 1, - "http://example.org/", - "field1=Text+1&field2=Text+2&submit=Submit+4"), - postResource( - msg, - 1, - "http://example.org/", - "field1=Text+1&field2=Text+2&submit=Submit+5"))); - } - - @Test - void shouldParsePostAndGetForms() { - // Given - org.zaproxy.zap.spider.parser.SpiderHtmlFormParser htmlParser = - createSpiderHtmlFormParser(); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage msg = createMessageWith("PostGetForms.html"); - Source source = createSource(msg); - // When - boolean completelyParsed = htmlParser.parseResource(msg, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfResourcesFound(), is(equalTo(6))); - assertThat( - listener.getResourcesFound(), - contains( - postResource( - msg, - 1, - "http://example.org/form1", - "field1=Text+1&field2=Text+2&submit=Submit"), - postResource( - msg, - 1, - "http://example.org/form1", - "field1=Text+1&field2=Text+2&submit=Submit+2"), - postResource( - msg, - 1, - "http://example.org/form1", - "field1=Text+1&field2=Text+2&submit3=Submit+3"), - uriResource(msg, 1, "http://example.org/form2?a=x&b=y&c=z"), - uriResource(msg, 1, "http://example.org/form2?a=x&b=y&submit=Submit+2"), - uriResource(msg, 1, "http://example.org/form2?a=x&b=y&submit3=Submit+3"))); - } - - @Test - void shouldNotParsePostFormIfPostFormProcessingIsDisabled() { - // Given - org.zaproxy.zap.spider.SpiderParam spiderOptions = createSpiderParamWithConfig(); - spiderOptions.setProcessForm(true); - spiderOptions.setPostForm(false); - org.zaproxy.zap.spider.parser.SpiderHtmlFormParser htmlParser = - new org.zaproxy.zap.spider.parser.SpiderHtmlFormParser( - spiderOptions, new org.zaproxy.zap.model.DefaultValueGenerator()); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage messageHtmlResponse = createMessageWith("POST", "Form.html"); - Source source = createSource(messageHtmlResponse); - // When - boolean completelyParsed = - htmlParser.parseResource(messageHtmlResponse, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(0))); - } - - @Test - void shouldParseNonPostFormIfPostFormProcessingIsDisabled() { - // Given - org.zaproxy.zap.spider.SpiderParam spiderOptions = createSpiderParamWithConfig(); - spiderOptions.setProcessForm(true); - spiderOptions.setPostForm(false); - org.zaproxy.zap.spider.parser.SpiderHtmlFormParser htmlParser = - new org.zaproxy.zap.spider.parser.SpiderHtmlFormParser( - spiderOptions, new org.zaproxy.zap.model.DefaultValueGenerator()); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage messageHtmlResponse = createMessageWith("GET", "Form.html"); - Source source = createSource(messageHtmlResponse); - // When - boolean completelyParsed = - htmlParser.parseResource(messageHtmlResponse, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(1))); - assertThat( - listener.getUrlsFound(), - contains("http://example.org/?field1=Text+1&field2=Text+2&submit=Submit")); - } - - @Test - void shouldParseFormAsGetIfNeitherGetNorPostForm() { - // Given - org.zaproxy.zap.spider.parser.SpiderHtmlFormParser htmlParser = - createSpiderHtmlFormParser(); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage messageHtmlResponse = createMessageWith("NonGetPostForm.html"); - Source source = createSource(messageHtmlResponse); - // When - boolean completelyParsed = - htmlParser.parseResource(messageHtmlResponse, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(1))); - assertThat( - listener.getUrlsFound(), - contains("http://example.org/?field1=Text+1&field2=Text+2&submit=Submit")); - } - - @Test - void shouldParseFormAsGetIfFormHasNoMethod() { - // Given - org.zaproxy.zap.spider.parser.SpiderHtmlFormParser htmlParser = - createSpiderHtmlFormParser(); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage messageHtmlResponse = createMessageWith("NoMethodForm.html"); - Source source = createSource(messageHtmlResponse); - // When - boolean completelyParsed = - htmlParser.parseResource(messageHtmlResponse, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(1))); - assertThat( - listener.getUrlsFound(), - contains("http://example.org/?field1=Text+1&field2=Text+2&submit=Submit")); - } - - @Test - void shouldParseFormAsGetIfFormHasNoMethodEvenIfPostFormProcessingIsDisabled() { - // Given - org.zaproxy.zap.spider.SpiderParam spiderOptions = createSpiderParamWithConfig(); - spiderOptions.setProcessForm(true); - spiderOptions.setPostForm(false); - org.zaproxy.zap.spider.parser.SpiderHtmlFormParser htmlParser = - new org.zaproxy.zap.spider.parser.SpiderHtmlFormParser( - spiderOptions, new org.zaproxy.zap.model.DefaultValueGenerator()); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage messageHtmlResponse = createMessageWith("NoMethodForm.html"); - Source source = createSource(messageHtmlResponse); - // When - boolean completelyParsed = - htmlParser.parseResource(messageHtmlResponse, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(1))); - assertThat( - listener.getUrlsFound(), - contains("http://example.org/?field1=Text+1&field2=Text+2&submit=Submit")); - } - - @Test - void shouldParseFormAsGetIfFormHasEmptyMethod() { - // Given - org.zaproxy.zap.spider.parser.SpiderHtmlFormParser htmlParser = - createSpiderHtmlFormParser(); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage messageHtmlResponse = createMessageWith("EmptyMethodForm.html"); - Source source = createSource(messageHtmlResponse); - // When - boolean completelyParsed = - htmlParser.parseResource(messageHtmlResponse, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(1))); - assertThat( - listener.getUrlsFound(), - contains("http://example.org/?field1=Text+1&field2=Text+2&submit=Submit")); - } - - @Test - void shouldUseMessageUrlAsActionIfFormHasNoAction() { - // Given - org.zaproxy.zap.spider.parser.SpiderHtmlFormParser htmlParser = - createSpiderHtmlFormParser(); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage messageHtmlResponse = createMessageWith("NoActionForm.html"); - Source source = createSource(messageHtmlResponse); - // When - boolean completelyParsed = - htmlParser.parseResource(messageHtmlResponse, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(1))); - assertThat( - listener.getUrlsFound(), - contains("http://example.com/?field1=Text+1&field2=Text+2&submit=Submit")); - } - - @Test - void shouldParseGetFormWithoutSubmitField() { - // Given - org.zaproxy.zap.spider.parser.SpiderHtmlFormParser htmlParser = - createSpiderHtmlFormParser(); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage messageHtmlResponse = createMessageWith("GET", "FormNoSubmitField.html"); - Source source = createSource(messageHtmlResponse); - // When - boolean completelyParsed = - htmlParser.parseResource(messageHtmlResponse, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(1))); - assertThat( - listener.getUrlsFound(), - contains("http://example.org/?field1=Text+1&field2=Text+2")); - } - - @Test - void shouldParsePostFormWithoutSubmitField() { - // Given - org.zaproxy.zap.spider.parser.SpiderHtmlFormParser htmlParser = - createSpiderHtmlFormParser(); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage msg = createMessageWith("POST", "FormNoSubmitField.html"); - Source source = createSource(msg); - // When - boolean completelyParsed = htmlParser.parseResource(msg, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(1))); - assertThat( - listener.getResourcesFound(), - contains( - postResource( - msg, 1, "http://example.org/", "field1=Text+1&field2=Text+2"))); - } - - @Test - void shouldRemoveFragmentFromActionWhenParsingGetForm() { - // Given - org.zaproxy.zap.spider.parser.SpiderHtmlFormParser htmlParser = - createSpiderHtmlFormParser(); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage messageHtmlResponse = createMessageWith("GET", "FormActionWithFragment.html"); - Source source = createSource(messageHtmlResponse); - // When - boolean completelyParsed = - htmlParser.parseResource(messageHtmlResponse, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(1))); - assertThat( - listener.getUrlsFound(), - contains("http://example.org/?field1=Text+1&field2=Text+2&submit=Submit")); - } - - @Test - void shouldRemoveFragmentFromActionWhenParsingPostForm() { - // Given - org.zaproxy.zap.spider.parser.SpiderHtmlFormParser htmlParser = - createSpiderHtmlFormParser(); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage messageHtmlResponse = createMessageWith("POST", "FormActionWithFragment.html"); - Source source = createSource(messageHtmlResponse); - // When - boolean completelyParsed = - htmlParser.parseResource(messageHtmlResponse, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(1))); - assertThat( - listener.getResourcesFound(), - contains( - postResource( - messageHtmlResponse, - 1, - "http://example.org/", - "field1=Text+1&field2=Text+2&submit=Submit"))); - } - - @Test - void shouldRemoveFragmentFromActionWhenParsingNeitherGetNorPostForm() { - // Given - org.zaproxy.zap.spider.parser.SpiderHtmlFormParser htmlParser = - createSpiderHtmlFormParser(); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage messageHtmlResponse = - createMessageWith("NeitherGetNorPost", "FormActionWithFragment.html"); - Source source = createSource(messageHtmlResponse); - // When - boolean completelyParsed = - htmlParser.parseResource(messageHtmlResponse, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(1))); - assertThat( - listener.getUrlsFound(), - contains("http://example.org/?field1=Text+1&field2=Text+2&submit=Submit")); - } - - @Test - void shouldAppendToEmptyQueryActionParametersWhenParsingGetForm() { - // Given - org.zaproxy.zap.spider.parser.SpiderHtmlFormParser htmlParser = - createSpiderHtmlFormParser(); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage messageHtmlResponse = createMessageWith("GetFormActionWithEmptyQuery.html"); - Source source = createSource(messageHtmlResponse); - // When - boolean completelyParsed = - htmlParser.parseResource(messageHtmlResponse, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(1))); - assertThat( - listener.getUrlsFound(), - contains("http://example.org/?field1=Text+1&field2=Text+2&submit=Submit")); - } - - @Test - void shouldAppendToQueryActionParametersWhenParsingGetForm() { - // Given - org.zaproxy.zap.spider.parser.SpiderHtmlFormParser htmlParser = - createSpiderHtmlFormParser(); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage messageHtmlResponse = createMessageWith("GetFormActionWithQuery.html"); - Source source = createSource(messageHtmlResponse); - // When - boolean completelyParsed = - htmlParser.parseResource(messageHtmlResponse, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(1))); - assertThat( - listener.getUrlsFound(), - contains("http://example.org/?a=b&c=d&field1=Text+1&field2=Text+2&submit=Submit")); - } - - @Test - void shouldAppendToQueryActionParametersTerminatedWithAmpersandWhenParsingGetForm() { - // Given - org.zaproxy.zap.spider.parser.SpiderHtmlFormParser htmlParser = - createSpiderHtmlFormParser(); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage messageHtmlResponse = createMessageWith("GetFormActionWithQueryAmpersand.html"); - Source source = createSource(messageHtmlResponse); - // When - boolean completelyParsed = - htmlParser.parseResource(messageHtmlResponse, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(1))); - assertThat( - listener.getUrlsFound(), - contains("http://example.org/?a=b&field1=Text+1&field2=Text+2&submit=Submit")); - } - - @Test - void shouldUseBaseHtmlUrlWhenParsingGetForm() { - // Given - org.zaproxy.zap.spider.parser.SpiderHtmlFormParser htmlParser = - createSpiderHtmlFormParser(); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage msg = - createMessageWith( - "GET", "FormWithHtmlBase.html", "search", "http://base.example.com/"); - Source source = createSource(msg); - // When - boolean completelyParsed = htmlParser.parseResource(msg, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(1))); - assertThat( - listener.getUrlsFound(), - contains("http://base.example.com/search?q=Search&submit=Submit")); - } - - @Test - void shouldUseAbsolutePathBaseHtmlUrlWhenParsingGetFormWithRelativeAction() { - // Given - org.zaproxy.zap.spider.parser.SpiderHtmlFormParser htmlParser = - createSpiderHtmlFormParser(); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage msg = - createMessageWith( - "GET", - "FormWithHtmlBase.html", - "action/relative", - "/base/absolute/path/", - "/a/b.html"); - Source source = createSource(msg); - // When - boolean completelyParsed = htmlParser.parseResource(msg, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(1))); - assertThat( - listener.getUrlsFound(), - contains( - "http://example.com/base/absolute/path/action/relative?q=Search&submit=Submit")); - } - - @Test - void shouldIgnoreAbsolutePathBaseHtmlUrlWhenParsingGetFormWithAbsoluteAction() { - // Given - org.zaproxy.zap.spider.parser.SpiderHtmlFormParser htmlParser = - createSpiderHtmlFormParser(); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage msg = - createMessageWith( - "GET", - "FormWithHtmlBase.html", - "/action/absolute", - "/base/absolute/path/", - "/a/b.html"); - Source source = createSource(msg); - // When - boolean completelyParsed = htmlParser.parseResource(msg, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(1))); - assertThat( - listener.getUrlsFound(), - contains("http://example.com/action/absolute?q=Search&submit=Submit")); - } - - @Test - void shouldUseRelativePathBaseHtmlUrlWhenParsingGetFormWithRelativeAction() { - // Given - org.zaproxy.zap.spider.parser.SpiderHtmlFormParser htmlParser = - createSpiderHtmlFormParser(); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage msg = - createMessageWith( - "GET", - "FormWithHtmlBase.html", - "action/relative", - "base/relative/path/", - "/a/b.html"); - Source source = createSource(msg); - // When - boolean completelyParsed = htmlParser.parseResource(msg, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(1))); - assertThat( - listener.getUrlsFound(), - contains( - "http://example.com/a/base/relative/path/action/relative?q=Search&submit=Submit")); - } - - @Test - void shouldUseButtonFormActionIfPresent() { - // Given - org.zaproxy.zap.spider.parser.SpiderHtmlFormParser htmlParser = - createSpiderHtmlFormParser(); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage msg = createMessageWith("GET", "FormWithFormactionButtons.html"); - Source source = createSource(msg); - // When - boolean completelyParsed = htmlParser.parseResource(msg, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(10))); - assertThat( - listener.getUrlsFound(), - contains( - "http://example.org/formaction1?field1=field1&field2=field2", - "http://example.org/form2?field1=field1&field2=field2", - "http://example.org/formaction2?field1=field1&field2=field2", - "http://example.org/emptyform", - "http://example.org/withchildbutton", - "http://example.org/withoutchildbutton", - "http://actionnoreset.com/", - "http://actionnobutton.com/", - "http://i.override.to.be.overridden.com/", - "http://not.overridden.by.nested.buttons.com/")); - } - - @Test - void shouldUseButtonFormMethodIfPresentGET() { - // Given - org.zaproxy.zap.spider.SpiderParam spiderOptions = createSpiderParamWithConfig(); - // Disable POST handling, we need just to retrieve the forms with GET methods - spiderOptions.setPostForm(false); - org.zaproxy.zap.spider.parser.SpiderHtmlFormParser htmlParser = - new org.zaproxy.zap.spider.parser.SpiderHtmlFormParser( - spiderOptions, new org.zaproxy.zap.model.DefaultValueGenerator()); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage msg = createMessageWith("GET", "OverriddenMethodByButtonForms.html"); - Source source = createSource(msg); - // When - boolean completelyParsed = htmlParser.parseResource(msg, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(4))); - assertThat( - listener.getUrlsFound(), - contains( - "http://example.org/form1?field1=Text+1&field2=Text+2", - "http://ignore.button.org/form3?a=x&b=y&c=z", - "http://ignore.reset.org/form4?a=x&b=y", - "http://example.org/form6?a=x&b=y")); - } - - @Test - void shouldUseButtonFormMethodIfPresentPOST() { - // Given - org.zaproxy.zap.spider.SpiderParam spiderOptions = createSpiderParamWithConfig(); - // Ensure POST handling is enabled, now both GET and POST methods should be identified by - // our code - spiderOptions.setPostForm(true); - // Disable POST handling, we need just to retrieve the forms with GET methods - org.zaproxy.zap.spider.parser.SpiderHtmlFormParser htmlParser = - new org.zaproxy.zap.spider.parser.SpiderHtmlFormParser( - spiderOptions, new org.zaproxy.zap.model.DefaultValueGenerator()); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage msg = createMessageWith("GET", "OverriddenMethodByButtonForms.html"); - Source source = createSource(msg); - // When - boolean completelyParsed = htmlParser.parseResource(msg, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(7))); - assertThat( - listener.getUrlsFound(), - contains( - "http://example.org/form1?field1=Text+1&field2=Text+2", - "http://example.org/form2", - "http://ignore.button.org/form3?a=x&b=y&c=z", - "http://ignore.reset.org/form4?a=x&b=y", - "http://outside.org/form5", - "http://example.org/form6?a=x&b=y", - "http://example.org/form7")); - } - - @Test - void shouldIgnoreRelativePathBaseHtmlUrlWhenParsingGetFormWithAbsoluteAction() { - // Given - org.zaproxy.zap.spider.parser.SpiderHtmlFormParser htmlParser = - createSpiderHtmlFormParser(); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage msg = - createMessageWith( - "GET", - "FormWithHtmlBase.html", - "/action/absolute", - "base/relative/path/", - "/a/b.html"); - Source source = createSource(msg); - // When - boolean completelyParsed = htmlParser.parseResource(msg, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(1))); - assertThat( - listener.getUrlsFound(), - contains("http://example.com/action/absolute?q=Search&submit=Submit")); - } - - @Test - void shouldIgnoreBaseHtmlIfEmptyHrefWhenParsingGetForm() { - // Given - org.zaproxy.zap.spider.parser.SpiderHtmlFormParser htmlParser = - createSpiderHtmlFormParser(); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage msg = createMessageWith("GET", "FormWithHtmlBase.html", "search", ""); - Source source = createSource(msg); - // When - boolean completelyParsed = htmlParser.parseResource(msg, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(1))); - assertThat( - listener.getUrlsFound(), - contains("http://example.com/search?q=Search&submit=Submit")); - } - - @Test - void shouldIgnoreBaseHtmlWithNoHrefWhenParsingGetForm() { - // Given - org.zaproxy.zap.spider.parser.SpiderHtmlFormParser htmlParser = - createSpiderHtmlFormParser(); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage msg = createMessageWith("GET", "FormWithHtmlBaseWithoutHref.html"); - Source source = createSource(msg); - // When - boolean completelyParsed = htmlParser.parseResource(msg, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(1))); - assertThat( - listener.getUrlsFound(), - contains("http://example.com/search?q=Search&submit=Submit")); - } - - @Test - void shouldIgnoreBaseHtmlIfActionIsAbsoluteWhenParsingGetForm() { - // Given - org.zaproxy.zap.spider.parser.SpiderHtmlFormParser htmlParser = - createSpiderHtmlFormParser(); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage msg = - createMessageWith( - "GET", - "FormWithHtmlBase.html", - "https://example.com/search", - "http://base.example.com/"); - Source source = createSource(msg); - // When - boolean completelyParsed = htmlParser.parseResource(msg, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(1))); - assertThat( - listener.getUrlsFound(), - contains("https://example.com/search?q=Search&submit=Submit")); - } - - @Test - void shouldUseBaseHtmlUrlWhenParsingPostForm() { - // Given - org.zaproxy.zap.spider.parser.SpiderHtmlFormParser htmlParser = - createSpiderHtmlFormParser(); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage msg = - createMessageWith( - "POST", "FormWithHtmlBase.html", "search", "http://base.example.com/"); - Source source = createSource(msg); - // When - boolean completelyParsed = htmlParser.parseResource(msg, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfResourcesFound(), is(equalTo(1))); - assertThat( - listener.getResourcesFound(), - contains( - postResource( - msg, - 1, - "http://base.example.com/search", - "q=Search&submit=Submit"))); - } - - @Test - void shouldUseAbsolutePathBaseHtmlUrlWhenParsingPostFormWithRelativeAction() { - // Given - org.zaproxy.zap.spider.parser.SpiderHtmlFormParser htmlParser = - createSpiderHtmlFormParser(); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage msg = - createMessageWith( - "POST", - "FormWithHtmlBase.html", - "action/relative", - "/base/absolute/path/", - "/a/b.html"); - Source source = createSource(msg); - // When - boolean completelyParsed = htmlParser.parseResource(msg, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfResourcesFound(), is(equalTo(1))); - assertThat( - listener.getResourcesFound(), - contains( - postResource( - msg, - 1, - "http://example.com/base/absolute/path/action/relative", - "q=Search&submit=Submit"))); - } - - @Test - void shouldIgnoreAbsolutePathBaseHtmlUrlWhenParsingPostFormWithAbsoluteAction() { - // Given - org.zaproxy.zap.spider.parser.SpiderHtmlFormParser htmlParser = - createSpiderHtmlFormParser(); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage msg = - createMessageWith( - "POST", - "FormWithHtmlBase.html", - "/action/absolute", - "/base/absolute/path/", - "/a/b.html"); - Source source = createSource(msg); - // When - boolean completelyParsed = htmlParser.parseResource(msg, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfResourcesFound(), is(equalTo(1))); - assertThat( - listener.getResourcesFound(), - contains( - postResource( - msg, - 1, - "http://example.com/action/absolute", - "q=Search&submit=Submit"))); - } - - @Test - void shouldUseRelativePathBaseHtmlUrlWhenParsingPostFormWithRelativeAction() { - // Given - org.zaproxy.zap.spider.parser.SpiderHtmlFormParser htmlParser = - createSpiderHtmlFormParser(); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage msg = - createMessageWith( - "POST", - "FormWithHtmlBase.html", - "action/relative", - "base/relative/path/", - "/a/b.html"); - Source source = createSource(msg); - // When - boolean completelyParsed = htmlParser.parseResource(msg, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfResourcesFound(), is(equalTo(1))); - assertThat( - listener.getResourcesFound(), - contains( - postResource( - msg, - 1, - "http://example.com/a/base/relative/path/action/relative", - "q=Search&submit=Submit"))); - } - - @Test - void shouldIgnoreRelativePathBaseHtmlUrlWhenParsingPostFormWithAbsoluteAction() { - // Given - org.zaproxy.zap.spider.parser.SpiderHtmlFormParser htmlParser = - createSpiderHtmlFormParser(); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage msg = - createMessageWith( - "POST", - "FormWithHtmlBase.html", - "/action/absolute", - "base/relative/path/", - "/a/b.html"); - Source source = createSource(msg); - // When - boolean completelyParsed = htmlParser.parseResource(msg, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfResourcesFound(), is(equalTo(1))); - assertThat( - listener.getResourcesFound(), - contains( - postResource( - msg, - 1, - "http://example.com/action/absolute", - "q=Search&submit=Submit"))); - } - - @Test - void shouldIgnoreBaseHtmlIfEmptyHrefWhenParsingPostForm() { - // Given - org.zaproxy.zap.spider.parser.SpiderHtmlFormParser htmlParser = - createSpiderHtmlFormParser(); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage msg = createMessageWith("POST", "FormWithHtmlBase.html", "search", ""); - Source source = createSource(msg); - // When - boolean completelyParsed = htmlParser.parseResource(msg, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfResourcesFound(), is(equalTo(1))); - assertThat( - listener.getResourcesFound(), - contains( - postResource( - msg, 1, "http://example.com/search", "q=Search&submit=Submit"))); - } - - @Test - void shouldIgnoreBaseHtmlWithNoHrefWhenParsingPostForm() { - // Given - org.zaproxy.zap.spider.parser.SpiderHtmlFormParser htmlParser = - createSpiderHtmlFormParser(); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage msg = createMessageWith("POST", "FormWithHtmlBaseWithoutHref.html"); - Source source = createSource(msg); - // When - boolean completelyParsed = htmlParser.parseResource(msg, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfResourcesFound(), is(equalTo(1))); - assertThat( - listener.getResourcesFound(), - contains( - postResource( - msg, 1, "http://example.com/search", "q=Search&submit=Submit"))); - } - - @Test - void shouldIgnoreBaseHtmlIfActionIsAbsoluteWhenParsingPostForm() { - // Given - org.zaproxy.zap.spider.parser.SpiderHtmlFormParser htmlParser = - createSpiderHtmlFormParser(); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage msg = - createMessageWith( - "POST", - "FormWithHtmlBase.html", - "https://example.com/search", - "http://base.example.com/"); - Source source = createSource(msg); - // When - boolean completelyParsed = htmlParser.parseResource(msg, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfResourcesFound(), is(equalTo(1))); - assertThat( - listener.getResourcesFound(), - contains( - postResource( - msg, 1, "https://example.com/search", "q=Search&submit=Submit"))); - } - - @Test - void shouldSetValuesToFieldsWithNoValueWhenParsingGetForm() { - // Given - org.zaproxy.zap.model.DefaultValueGenerator valueGenerator = - new org.zaproxy.zap.model.DefaultValueGenerator(); - org.zaproxy.zap.spider.parser.SpiderHtmlFormParser htmlParser = - createSpiderHtmlFormParser(valueGenerator); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - Date date = new Date(1474370354555L); - valueGenerator.setDefaultDate(date); - HttpMessage msg = createMessageWith("GET", "FormNoDefaultValues.html"); - Source source = createSource(msg); - // When - boolean completelyParsed = htmlParser.parseResource(msg, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfResourcesFound(), is(equalTo(8))); - assertThat( - listener.getUrlsFound(), - contains( - "http://example.org/?_file=test_file.txt&_hidden&_no-type=ZAP&_password=ZAP&_text=ZAP&submit=Submit", - "http://example.org/html5/number?_number=1&_number-max=2&_number-min=1&submit=Submit", - "http://example.org/html5/range?_range=1&_range-max=4&_range-min=3&submit=Submit", - "http://example.org/html5/misc?_color=%23ffffff&_email=foo-bar%40example.com&_tel=9999999999&_url=http%3A%2F%2Fwww.example.com&submit=Submit", - "http://example.org/unknown?_unknown&submit=Submit", - "http://example.org/selects?_select-one-option=first-option&_select-selected-option=selected-option&_select-two-options=last-option&submit=Submit", - "http://example.org/checkbox?_checkbox=first-checkbox&submit=Submit", - "http://example.org/html5/date-time?" - + params( - param("_date", formattedDate("yyyy-MM-dd", date)), - param( - "_datetime", - formattedDate("yyyy-MM-dd'T'HH:mm:ss'Z'", date)), - param( - "_datetime-local", - formattedDate("yyyy-MM-dd'T'HH:mm:ss", date)), - param("_month", formattedDate("yyyy-MM", date)), - param("_time", formattedDate("HH:mm:ss", date)), - param("_week", formattedDate("yyyy-'W'ww", date)), - param("submit", "Submit")))); - } - - @Test - void shouldSetValuesToFieldsWithNoValueWhenParsingPostForm() { - // Given - org.zaproxy.zap.model.DefaultValueGenerator valueGenerator = - new org.zaproxy.zap.model.DefaultValueGenerator(); - org.zaproxy.zap.spider.parser.SpiderHtmlFormParser htmlParser = - createSpiderHtmlFormParser(valueGenerator); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - Date date = new Date(1474370354555L); - valueGenerator.setDefaultDate(date); - HttpMessage msg = createMessageWith("POST", "FormNoDefaultValues.html"); - Source source = createSource(msg); - // When - boolean completelyParsed = htmlParser.parseResource(msg, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfResourcesFound(), is(equalTo(8))); - assertThat( - listener.getResourcesFound(), - contains( - postResource( - msg, - 1, - "http://example.org/", - "_hidden=&_no-type=ZAP&_text=ZAP&_password=ZAP&_file=test_file.txt&submit=Submit"), - postResource( - msg, - 1, - "http://example.org/html5/number", - "_number=1&_number-min=1&_number-max=2&submit=Submit"), - postResource( - msg, - 1, - "http://example.org/html5/range", - "_range=1&_range-min=3&_range-max=4&submit=Submit"), - postResource( - msg, - 1, - "http://example.org/html5/misc", - "_url=http%3A%2F%2Fwww.example.com&_email=foo-bar%40example.com&_color=%23ffffff&_tel=9999999999&submit=Submit"), - postResource( - msg, 1, "http://example.org/unknown", "_unknown=&submit=Submit"), - postResource( - msg, - 1, - "http://example.org/selects", - "_select-one-option=first-option&_select-two-options=last-option&_select-selected-option=selected-option&submit=Submit"), - postResource( - msg, - 1, - "http://example.org/checkbox", - "_checkbox=first-checkbox&submit=Submit"), - postResource( - msg, - 1, - "http://example.org/html5/date-time", - params( - param( - "_datetime", - formattedDate("yyyy-MM-dd'T'HH:mm:ss'Z'", date)), - param( - "_datetime-local", - formattedDate("yyyy-MM-dd'T'HH:mm:ss", date)), - param("_date", formattedDate("yyyy-MM-dd", date)), - param("_time", formattedDate("HH:mm:ss", date)), - param("_month", formattedDate("yyyy-MM", date)), - param("_week", formattedDate("yyyy-'W'ww", date)), - param("submit", "Submit"))))); - } - - @Test - void shouldProvidedCorrectFormDataToValueGenerator() { - // Given - TestValueGenerator valueGenerator = new TestValueGenerator(); - org.zaproxy.zap.spider.parser.SpiderHtmlFormParser htmlParser = - createSpiderHtmlFormParser(valueGenerator); - HttpMessage msg = createMessageWith("FormsForValueGenerator.html"); - Source source = createSource(msg); - int fieldIndex = 0; - // When - htmlParser.parseResource(msg, source, BASE_DEPTH); - // Then - assertThat(valueGenerator.getFields(), hasSize(9)); - assertThat( - valueGenerator.getFields().get(fieldIndex), - is( - equalTo( - formField( - "http://example.com/", - "http://example.org/post", - "field1", - "preDefValue1", - list(""), - attributes( - attribute("name", "field1"), - attribute("value", "preDefValue1"), - attribute("type", "hidden"), - attribute("id", "id1"), - attribute("Control Type", "HIDDEN")), - attributes( - attribute("action", "http://example.org/post"), - attribute("method", "POST"), - attribute("atta", "valueA")))))); - fieldIndex++; - assertThat( - valueGenerator.getFields().get(fieldIndex), - is( - equalTo( - formField( - "http://example.com/", - "http://example.org/post", - "field2", - "preDefValue2", - list(""), - attributes( - attribute("name", "field2"), - attribute("value", "preDefValue2"), - attribute("id", "id2"), - attribute("att1", "value1"), - attribute("Control Type", "TEXT")), - attributes( - attribute("action", "http://example.org/post"), - attribute("method", "POST"), - attribute("atta", "valueA")))))); - fieldIndex++; - assertThat( - valueGenerator.getFields().get(fieldIndex), - is( - equalTo( - formField( - "http://example.com/", - "http://example.org/post", - "field3", - "preDefValue3", - list(""), - attributes( - attribute("name", "field3"), - attribute("value", "preDefValue3"), - attribute("type", "text"), - attribute("Control Type", "TEXT")), - attributes( - attribute("action", "http://example.org/post"), - attribute("method", "POST"), - attribute("atta", "valueA")))))); - fieldIndex++; - assertThat( - valueGenerator.getFields().get(fieldIndex), - is( - equalTo( - formField( - "http://example.com/", - "http://example.org/post", - "gender", - "m", - list(("m,f")), - attributes( - attribute("name", "gender"), - attribute("type", "radio"), - attribute("value", "m"), - attribute("id", "male"), - attribute("Control Type", "RADIO")), - attributes( - attribute("action", "http://example.org/post"), - attribute("method", "POST"), - attribute("atta", "valueA")))))); - fieldIndex++; - assertThat( - valueGenerator.getFields().get(fieldIndex), - is( - equalTo( - formField( - "http://example.com/", - "http://example.org/post", - "submit", - "Submit", - list(""), - attributes( - attribute("name", "submit"), - attribute("type", "submit"), - attribute("value", "Submit"), - attribute("Control Type", "SUBMIT")), - attributes( - attribute("action", "http://example.org/post"), - attribute("method", "POST"), - attribute("atta", "valueA")))))); - fieldIndex++; - assertThat( - valueGenerator.getFields().get(fieldIndex), - is( - equalTo( - formField( - "http://example.com/", - "http://example.org/get", - "field1", - "", - list(""), - attributes( - attribute("name", "field1"), - attribute("type", "hidden"), - attribute("id", "id1"), - attribute("Control Type", "HIDDEN")), - attributes( - attribute("action", "http://example.org/get"), - attribute("method", "GET"), - attribute("att1", "value1"), - attribute("att2", "value2")))))); - fieldIndex++; - assertThat( - valueGenerator.getFields().get(fieldIndex), - is( - equalTo( - formField( - "http://example.com/", - "http://example.org/get", - "field2", - "", - list(""), - attributes( - attribute("name", "field2"), - attribute("id", "id2"), - attribute("att1", "value1"), - attribute("Control Type", "TEXT")), - attributes( - attribute("action", "http://example.org/get"), - attribute("method", "GET"), - attribute("att1", "value1"), - attribute("att2", "value2")))))); - fieldIndex++; - assertThat( - valueGenerator.getFields().get(fieldIndex), - is( - equalTo( - formField( - "http://example.com/", - "http://example.org/get", - "field3", - "", - list(""), - attributes( - attribute("name", "field3"), - attribute("type", "text"), - attribute("Control Type", "TEXT")), - attributes( - attribute("action", "http://example.org/get"), - attribute("method", "GET"), - attribute("att1", "value1"), - attribute("att2", "value2")))))); - fieldIndex++; - assertThat( - valueGenerator.getFields().get(fieldIndex), - is( - equalTo( - formField( - "http://example.com/", - "http://example.org/get", - "submit", - "Submit", - list(""), - attributes( - attribute("name", "submit"), - attribute("type", "submit"), - attribute("value", "Submit"), - attribute("Control Type", "SUBMIT")), - attributes( - attribute("action", "http://example.org/get"), - attribute("method", "GET"), - attribute("att1", "value1"), - attribute("att2", "value2")))))); - } - - @Test - void shouldParseGetFormAndIncludeRelatedInputsWithFormAttribute() { - // Given - org.zaproxy.zap.spider.parser.SpiderHtmlFormParser htmlParser = - createSpiderHtmlFormParser(); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage messageHtmlResponse = - createMessageWith("GET", "FormAndInputsWithFormAttributes.html"); - Source source = createSource(messageHtmlResponse); - // When - boolean completelyParsed = - htmlParser.parseResource(messageHtmlResponse, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(3))); - assertThat( - listener.getUrlsFound(), - contains( - "http://example.org/?field1=Field1&field2=Field2&field3=Field3&field4=Field4&submit=Submit1", - "http://example.org/?field1=Field1&field2=Field2&field3=Field3&field4=Field4&submit=Submit2", - "http://example.org/?field1=Field1&field2=Field2&field3=Field3&field4=Field4&submit=Submit3")); - } - - @Test - void shouldParsePostFormAndIncludeRelatedInputsWithFormAttribute() { - // Given - org.zaproxy.zap.spider.parser.SpiderHtmlFormParser htmlParser = - createSpiderHtmlFormParser(); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage msg = createMessageWith("POST", "FormAndInputsWithFormAttributes.html"); - Source source = createSource(msg); - // When - boolean completelyParsed = htmlParser.parseResource(msg, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(3))); - assertThat( - listener.getResourcesFound(), - contains( - postResource( - msg, - 1, - "http://example.org/", - "field1=Field1&submit=Submit1&field2=Field2&field3=Field3&field4=Field4"), - postResource( - msg, - 1, - "http://example.org/", - "field1=Field1&submit=Submit2&field2=Field2&field3=Field3&field4=Field4"), - postResource( - msg, - 1, - "http://example.org/", - "field1=Field1&submit=Submit3&field2=Field2&field3=Field3&field4=Field4"))); - } - - private static String formattedDate(String format, Date date) { - return new SimpleDateFormat(format).format(date); - } - - private static org.zaproxy.zap.spider.parser.SpiderHtmlFormParser createSpiderHtmlFormParser() { - return createSpiderHtmlFormParser(new org.zaproxy.zap.model.DefaultValueGenerator()); - } - - private static org.zaproxy.zap.spider.parser.SpiderHtmlFormParser createSpiderHtmlFormParser( - org.zaproxy.zap.model.ValueGenerator valueGenerator) { - org.zaproxy.zap.spider.SpiderParam spiderOptions = createSpiderParamWithConfig(); - spiderOptions.setProcessForm(true); - spiderOptions.setPostForm(true); - return new org.zaproxy.zap.spider.parser.SpiderHtmlFormParser( - spiderOptions, valueGenerator); - } - - private static HttpMessage createMessageWith(String filename) { - return createMessageWith(null, filename); - } - - private static HttpMessage createMessageWith(String formMethod, String filename) { - return createMessageWith(formMethod, filename, null, null, "/"); - } - - private static HttpMessage createMessageWith( - String formMethod, String filename, String formAction, String baseHtml) { - return createMessageWith(formMethod, filename, formAction, baseHtml, "/"); - } - - private static HttpMessage createMessageWith( - String formMethod, - String filename, - String formAction, - String baseHtml, - String requestUri) { - HttpMessage message = new HttpMessage(); - try { - String fileContents = readFile(BASE_DIR_HTML_FILES.resolve(filename)); - if (formMethod != null) { - fileContents = fileContents.replace(FORM_METHOD_TOKEN, formMethod); - } - if (formAction != null) { - fileContents = fileContents.replace(FORM_ACTION_TOKEN, formAction); - } - if (baseHtml != null) { - fileContents = fileContents.replace(BASE_HTML_TOKEN, baseHtml); - } - message.setRequestHeader("GET " + requestUri + " HTTP/1.1\r\nHost: example.com\r\n"); - message.setResponseHeader( - "HTTP/1.1 200 OK\r\n" - + "Content-Type: text/html; charset=UTF-8\r\n" - + "Content-Length: " - + fileContents.length()); - message.setResponseBody(fileContents); - } catch (Exception e) { - throw new RuntimeException(e); - } - return message; - } - - private static class TestValueGenerator implements org.zaproxy.zap.model.ValueGenerator { - - private final List fields; - - TestValueGenerator() { - fields = new ArrayList<>(); - } - - List getFields() { - return fields; - } - - @Override - public String getValue( - URI uri, - String targetUri, - String fieldName, - String defaultValue, - List values, - Map formAttributes, - Map fieldAttributes) { - fields.add( - new FormField( - uri.toString(), - targetUri, - fieldName, - defaultValue, - values, - fieldAttributes, - formAttributes)); - return ""; - } - } - - private static class FormField { - - private final String uri; - private final String targetUri; - private final String fieldName; - private final String defaultValue; - private final List values; - private final Map fieldAttributes; - private final Map formAttributes; - - FormField( - String uri, - String targetUri, - String fieldName, - String defaultValue, - List values, - Map fieldAttributes, - Map formAttributes) { - this.uri = uri; - this.targetUri = targetUri; - this.fieldName = fieldName; - this.defaultValue = defaultValue; - this.values = values; - this.fieldAttributes = new HashMap<>(fieldAttributes); - this.formAttributes = new HashMap<>(formAttributes); - } - - String getUri() { - return uri; - } - - String getTargetUri() { - return targetUri; - } - - String getDefaultValue() { - return defaultValue; - } - - String getFieldName() { - return fieldName; - } - - List getValues() { - return values; - } - - Map getFieldAttributes() { - return fieldAttributes; - } - - Map getFormAttributes() { - return formAttributes; - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((fieldAttributes == null) ? 0 : fieldAttributes.hashCode()); - result = prime * result + ((fieldName == null) ? 0 : fieldName.hashCode()); - result = prime * result + ((defaultValue == null) ? 0 : defaultValue.hashCode()); - result = prime * result + ((values == null) ? 0 : values.hashCode()); - result = prime * result + ((formAttributes == null) ? 0 : formAttributes.hashCode()); - result = prime * result + ((targetUri == null) ? 0 : targetUri.hashCode()); - result = prime * result + ((uri == null) ? 0 : uri.hashCode()); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null) { - return false; - } - if (getClass() != obj.getClass()) { - return false; - } - FormField other = (FormField) obj; - if (fieldAttributes == null) { - if (other.fieldAttributes != null) { - return false; - } - } else if (!fieldAttributes.equals(other.fieldAttributes)) { - return false; - } - if (fieldName == null) { - if (other.fieldName != null) { - return false; - } - } else if (!fieldName.equals(other.fieldName)) { - return false; - } - if (defaultValue == null) { - if (other.defaultValue != null) { - return false; - } - } else if (!defaultValue.equals(other.defaultValue)) { - return false; - } - if (values == null) { - if (other.values != null) { - return false; - } - } else if (!values.equals(other.values)) { - return false; - } - if (formAttributes == null) { - if (other.formAttributes != null) { - return false; - } - } else if (!formAttributes.equals(other.formAttributes)) { - return false; - } - if (targetUri == null) { - if (other.targetUri != null) { - return false; - } - } else if (!targetUri.equals(other.targetUri)) { - return false; - } - if (uri == null) { - if (other.uri != null) { - return false; - } - } else if (!uri.equals(other.uri)) { - return false; - } - return true; - } - - @Override - public String toString() { - StringBuilder strBuilder = new StringBuilder(250); - strBuilder.append("uri=").append(uri); - strBuilder.append(", targetUri=").append(targetUri); - strBuilder.append(", fieldName=").append(fieldName); - strBuilder.append(", defaultValue=").append(defaultValue); - strBuilder.append(", values=").append(values); - strBuilder.append(", fieldAttributes=").append(fieldAttributes); - strBuilder.append(", formAttributes=").append(formAttributes); - return strBuilder.toString(); - } - } - - private static FormField formField( - String uri, - String targetUri, - String fieldName, - String defaultValue, - List values, - Map fieldAttributes, - Map formAttributes) { - return new FormField( - uri, targetUri, fieldName, defaultValue, values, fieldAttributes, formAttributes); - } - - @SafeVarargs - private static Map attributes(Pair... attributes) { - if (attributes == null || attributes.length == 0) { - return Collections.emptyMap(); - } - - Map mapAttributes = new HashMap<>(); - for (Pair attribute : attributes) { - mapAttributes.put(attribute.first, attribute.second); - } - return mapAttributes; - } - - private static Pair attribute(String name, String value) { - return new Pair<>(name, value); - } - - private static List list(String preDefValue) { - if (preDefValue == null || preDefValue.isEmpty()) { - return new ArrayList<>(); - } - List values = new ArrayList<>(); - String[] value = preDefValue.split(","); - for (String val : value) { - values.add(val); - } - return values; - } -} diff --git a/zap/src/test/java/org/zaproxy/zap/spider/parser/SpiderHtmlParserUnitTest.java b/zap/src/test/java/org/zaproxy/zap/spider/parser/SpiderHtmlParserUnitTest.java deleted file mode 100644 index ee9e40a8100..00000000000 --- a/zap/src/test/java/org/zaproxy/zap/spider/parser/SpiderHtmlParserUnitTest.java +++ /dev/null @@ -1,1012 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2016 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.spider.parser; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.contains; -import static org.hamcrest.Matchers.empty; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; -import static org.junit.jupiter.api.Assertions.assertThrows; - -import java.nio.file.Path; -import net.htmlparser.jericho.Source; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; -import org.parosproxy.paros.network.HttpMessage; - -/** Unit test for {@link SpiderHtmlParser}. */ -@SuppressWarnings("deprecation") -class SpiderHtmlParserUnitTest extends SpiderParserTestUtils { - - private static final String ROOT_PATH = "/"; - private static final int BASE_DEPTH = 0; - - private static final Path BASE_DIR_HTML_FILES = - getResourcePath("html", SpiderHtmlParserUnitTest.class); - - @Test - void shouldFailToCreateParserWithUndefinedSpiderOptions() { - // Given - org.zaproxy.zap.spider.SpiderParam undefinedSpiderOptions = null; - // When / Then - assertThrows( - NullPointerException.class, () -> new SpiderHtmlParser(undefinedSpiderOptions)); - } - - @Test - void shouldFailToEvaluateAnUndefinedMessage() { - // Given - HttpMessage undefinedMessage = null; - SpiderHtmlParser htmlParser = - new SpiderHtmlParser(new org.zaproxy.zap.spider.SpiderParam()); - // When / Then - assertThrows( - NullPointerException.class, - () -> htmlParser.canParseResource(undefinedMessage, ROOT_PATH, false)); - } - - @Test - void shouldParseHtmlResponse() { - // Given - SpiderHtmlParser htmlParser = - new SpiderHtmlParser(new org.zaproxy.zap.spider.SpiderParam()); - HttpMessage messageHtmlResponse = createMessageWith("NoURLsSpiderHtmlParser.html"); - boolean parsed = false; - // When - boolean canParse = htmlParser.canParseResource(messageHtmlResponse, ROOT_PATH, parsed); - // Then - assertThat(canParse, is(equalTo(true))); - } - - @Test - void shouldParseHtmlResponseEvenIfProvidedPathIsNull() { - // Given - SpiderHtmlParser htmlParser = - new SpiderHtmlParser(new org.zaproxy.zap.spider.SpiderParam()); - HttpMessage messageHtmlResponse = createMessageWith("NoURLsSpiderHtmlParser.html"); - boolean parsed = false; - // When - boolean canParse = htmlParser.canParseResource(messageHtmlResponse, null, parsed); - // Then - assertThat(canParse, is(equalTo(true))); - } - - @Test - void shouldNotParseHtmlResponseIfAlreadyParsed() { - // Given - SpiderHtmlParser htmlParser = - new SpiderHtmlParser(new org.zaproxy.zap.spider.SpiderParam()); - HttpMessage messageHtmlResponse = createMessageWith("NoURLsSpiderHtmlParser.html"); - boolean parsed = true; - // When - boolean canParse = htmlParser.canParseResource(messageHtmlResponse, ROOT_PATH, parsed); - // Then - assertThat(canParse, is(equalTo(false))); - } - - @Test - void shouldFailToParseAnUndefinedMessage() { - // Given - HttpMessage undefinedMessage = null; - SpiderHtmlParser htmlParser = - new SpiderHtmlParser(new org.zaproxy.zap.spider.SpiderParam()); - Source source = createSource(createMessageWith("NoURLsSpiderHtmlParser.html")); - // When / Then - assertThrows( - NullPointerException.class, - () -> htmlParser.parseResource(undefinedMessage, source, BASE_DEPTH)); - } - - @Test - void shouldParseMessageEvenWithoutSource() { - // Given - Source source = null; - SpiderHtmlParser htmlParser = - new SpiderHtmlParser(new org.zaproxy.zap.spider.SpiderParam()); - HttpMessage messageHtmlResponse = createMessageWith("NoURLsSpiderHtmlParser.html"); - // When / Then - assertDoesNotThrow(() -> htmlParser.parseResource(messageHtmlResponse, source, BASE_DEPTH)); - } - - @Test - void shouldNeverConsiderCompletelyParsed() { - // Given - Source source = null; - SpiderHtmlParser htmlParser = - new SpiderHtmlParser(new org.zaproxy.zap.spider.SpiderParam()); - HttpMessage messageHtmlResponse = createMessageWith("NoURLsSpiderHtmlParser.html"); - // When - boolean completelyParsed = - htmlParser.parseResource(messageHtmlResponse, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - } - - @Test - void shouldFindUrlsInAElements() { - // Given - SpiderHtmlParser htmlParser = - new SpiderHtmlParser(new org.zaproxy.zap.spider.SpiderParam()); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage messageHtmlResponse = createMessageWith("AElementsSpiderHtmlParser.html"); - Source source = createSource(messageHtmlResponse); - // When - boolean completelyParsed = - htmlParser.parseResource(messageHtmlResponse, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(7))); - assertThat( - listener.getUrlsFound(), - contains( - "http://a.example.com/base/scheme", - "http://a.example.com:8000/b", - "https://a.example.com/c?a=b", - "http://example.com/sample/a/relative", - "http://example.com/sample/", - "http://example.com/a/absolute", - "ftp://a.example.com/")); - } - - @Test - void shouldFindUrlsInAnchorPingElements() { - // Given - SpiderHtmlParser htmlParser = - new SpiderHtmlParser(new org.zaproxy.zap.spider.SpiderParam()); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage messageHtmlResponse = - createMessageWith("AElementsWithPingSpiderHtmlParser.html"); - Source source = createSource(messageHtmlResponse); - // When - boolean completelyParsed = - htmlParser.parseResource(messageHtmlResponse, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(23))); - assertThat( - listener.getUrlsFound(), - contains( - // a URLs followed by ping URLs - "http://a.example.com/base/scheme", - "http://ping.example.com/base/scheme", - "http://a.example.com:8000/b", - "http://ping.example.com:8000/b", - "https://a.example.com/c?a=b", - "https://ping.example.com/c?a=b", - "http://example.com/sample/a/relative", - "http://example.com/sample/a/relative/ping", - "http://example.com/a/absolute", - "http://example.com/a/absolute/ping", - "ftp://a.example.com/", - "https://ping.example.com/ping", - // Ping first, is parsed href before ping - "http://b.example.com/", - "https://ping.first.com/", - // Ignored anchors but picked pings - "http://ping.example.com/mailping", - "http://ping.example.com/jsping", - "http://ping.example.com/ping", - // Multiple ping URLs - "http://a.example.com/", - "http://ping.example.com/", - "http://pong.example.com/", - // Multiple ping URLs with tab in the middle - "http://a.example.com/", - "http://ping.example.com/", - "http://pong.example.com/")); // Trailing slash is added on host - } - - @Test - void shouldFindUrlsInAppletElements() { - // Given - SpiderHtmlParser htmlParser = - new SpiderHtmlParser(new org.zaproxy.zap.spider.SpiderParam()); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage messageHtmlResponse = createMessageWith("AppletElementsSpiderHtmlParser.html"); - Source source = createSource(messageHtmlResponse); - // When - boolean completelyParsed = - htmlParser.parseResource(messageHtmlResponse, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(8))); - assertThat( - listener.getUrlsFound(), - contains( - "http://example.com/applet/src/file.class", - "https://example.com/applet/src/file.class", - "https://example.com/absolute/codebase/", - "http://example.com/sample/relative/applet/codebase/", - "http://example.com/absolute/applet/codebase/", - "http://example.com/test/html/body/applet/archive", - "https://example.com/absolute/archive", - "http://example.com/archive.zip")); - } - - @Test - void shouldFindUrlsInImportElements() { - // Given - SpiderHtmlParser htmlParser = - new SpiderHtmlParser(new org.zaproxy.zap.spider.SpiderParam()); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage messageHtmlResponse = createMessageWith("ImportElementsSpiderHtmlParser.html"); - Source source = createSource(messageHtmlResponse); - // When - boolean completelyParsed = - htmlParser.parseResource(messageHtmlResponse, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(5))); - assertThat( - listener.getUrlsFound(), - contains( - "http://example.com/import/namespace/implementation", - "https://import.example.com:9000/namespace/implementation", - "http://import.example.com/namespace/implementation", - "http://example.com/sample/import/namespace/implementation", - "ftp://import.example.com/namespace/implementation")); - } - - @Test - void shouldFindUrlsInAreaPingElements() { - // Given - SpiderHtmlParser htmlParser = - new SpiderHtmlParser(new org.zaproxy.zap.spider.SpiderParam()); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage messageHtmlResponse = - createMessageWith("AreaElementsWithPingSpiderHtmlParser.html"); - Source source = createSource(messageHtmlResponse); - // When - boolean completelyParsed = - htmlParser.parseResource(messageHtmlResponse, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(23))); - assertThat( - listener.getUrlsFound(), - contains( - // area URLs followed by ping URLs - "http://a.example.com/base/scheme", - "http://ping.example.com/base/scheme", - "http://a.example.com:8000/b", - "http://ping.example.com:8000/b", - "https://a.example.com/c?a=b", - "https://ping.example.com/c?a=b", - "http://example.com/sample/a/relative", - "http://example.com/sample/a/relative/ping", - "http://example.com/a/absolute", - "http://example.com/a/absolute/ping", - "ftp://a.example.com/", - "https://ping.example.com/ping", - // Ping first, is parsed href before ping - "http://b.example.com/", - "https://ping.first.com/", - // Ignored anchors but picked pings - "http://ping.example.com/mailping", - "http://ping.example.com/jsping", - "http://ping.example.com/ping", - // Multiple ping URLs - "http://a.example.com/", - "http://ping.example.com/", - "http://pong.example.com/", - // Multiple ping URLs with tab in the middle - "http://a.example.com/", - "http://ping.example.com/", - "http://pong.example.com/")); // Trailing slash is added on host - } - - @Test - void shouldUseMessageUriIfNoBaseElement() { - // Given - SpiderHtmlParser htmlParser = - new SpiderHtmlParser(new org.zaproxy.zap.spider.SpiderParam()); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage messageHtmlResponse = - createMessageWith("NoBaseWithAElementSpiderHtmlParser.html"); - Source source = createSource(messageHtmlResponse); - // When - boolean completelyParsed = - htmlParser.parseResource(messageHtmlResponse, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(1))); - assertThat(listener.getUrlsFound(), contains("http://example.com/relative/no/base")); - } - - @Test - void shouldUseAbsolutePathBaseElement() { - // Given - SpiderHtmlParser htmlParser = - new SpiderHtmlParser(new org.zaproxy.zap.spider.SpiderParam()); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage messageHtmlResponse = - createMessageWith("BaseWithAbsolutePathHrefAElementSpiderHtmlParser.html", "/a/b"); - Source source = createSource(messageHtmlResponse); - // When - boolean completelyParsed = - htmlParser.parseResource(messageHtmlResponse, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(2))); - assertThat( - listener.getUrlsFound(), - contains( - "http://example.com/base/absolute/path/relative/a/element", - "http://example.com/absolute/a/element")); - } - - @Test - void shouldUseRelativePathBaseElement() { - // Given - SpiderHtmlParser htmlParser = - new SpiderHtmlParser(new org.zaproxy.zap.spider.SpiderParam()); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage messageHtmlResponse = - createMessageWith("BaseWithRelativePathHrefAElementSpiderHtmlParser.html", "/a/b"); - Source source = createSource(messageHtmlResponse); - // When - boolean completelyParsed = - htmlParser.parseResource(messageHtmlResponse, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(2))); - assertThat( - listener.getUrlsFound(), - contains( - "http://example.com/a/base/relative/path/relative/a/element", - "http://example.com/absolute/a/element")); - } - - @Test - void shouldIgnoreBaseAndUseMessageUriIfBaseElementDoesNotHaveHref() { - // Given - SpiderHtmlParser htmlParser = - new SpiderHtmlParser(new org.zaproxy.zap.spider.SpiderParam()); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage messageHtmlResponse = - createMessageWith("BaseWithoutHrefAElementSpiderHtmlParser.html"); - Source source = createSource(messageHtmlResponse); - // When - boolean completelyParsed = - htmlParser.parseResource(messageHtmlResponse, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(1))); - assertThat(listener.getUrlsFound(), contains("http://example.com/relative/no/base")); - } - - @Test - void shouldIgnoreBaseAndUseMessageUriIfBaseElementHaveEmptyHref() { - // Given - SpiderHtmlParser htmlParser = - new SpiderHtmlParser(new org.zaproxy.zap.spider.SpiderParam()); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage messageHtmlResponse = - createMessageWith("BaseWithEmptyHrefAElementSpiderHtmlParser.html"); - Source source = createSource(messageHtmlResponse); - // When - boolean completelyParsed = - htmlParser.parseResource(messageHtmlResponse, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(1))); - assertThat(listener.getUrlsFound(), contains("http://example.com/relative/no/base")); - } - - @Test - void shouldFindUrlsInAreaElements() throws Exception { - // Given - SpiderHtmlParser htmlParser = - new SpiderHtmlParser(new org.zaproxy.zap.spider.SpiderParam()); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage messageHtmlResponse = createMessageWith("AreaElementsSpiderHtmlParser.html"); - Source source = createSource(messageHtmlResponse); - // When - boolean completelyParsed = - htmlParser.parseResource(messageHtmlResponse, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(7))); - assertThat( - listener.getUrlsFound(), - contains( - "http://area.example.com/base/scheme", - "http://area.example.com:8000/b", - "https://area.example.com/c?a=b", - "http://example.com/sample/area/relative", - "http://example.com/sample/", - "http://example.com/area/absolute", - "ftp://area.example.com/")); - } - - @Test - void shouldFindUrlsInAudioElements() throws Exception { - // Given - SpiderHtmlParser htmlParser = - new SpiderHtmlParser(new org.zaproxy.zap.spider.SpiderParam()); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage messageHtmlResponse = createMessageWith("AudioElementsSpiderHtmlParser.html"); - Source source = createSource(messageHtmlResponse); - // When - boolean completelyParsed = - htmlParser.parseResource(messageHtmlResponse, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(3))); - assertThat( - listener.getUrlsFound(), - contains( - "http://example.com/sample/relative/src", - "http://example.com/absolute/src", - "https://audio.example.com/external/audio/src")); - } - - @Test - void shouldFindUrlsInEmbedElements() { - // Given - SpiderHtmlParser htmlParser = - new SpiderHtmlParser(new org.zaproxy.zap.spider.SpiderParam()); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage messageHtmlResponse = createMessageWith("EmbedElementsSpiderHtmlParser.html"); - Source source = createSource(messageHtmlResponse); - // When - boolean completelyParsed = - htmlParser.parseResource(messageHtmlResponse, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(7))); - assertThat( - listener.getUrlsFound(), - contains( - "http://embed.example.com/base/scheme", - "http://embed.example.com:8000/b", - "https://embed.example.com/c?a=b", - "http://example.com/sample/embed/relative", - "http://example.com/sample/", - "http://example.com/embed/absolute", - "ftp://embed.example.com/")); - } - - @Test - void shouldFindUrlsInFrameElements() { - // Given - SpiderHtmlParser htmlParser = - new SpiderHtmlParser(new org.zaproxy.zap.spider.SpiderParam()); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage messageHtmlResponse = createMessageWith("FrameElementsSpiderHtmlParser.html"); - Source source = createSource(messageHtmlResponse); - // When - boolean completelyParsed = - htmlParser.parseResource(messageHtmlResponse, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(7))); - assertThat( - listener.getUrlsFound(), - contains( - "http://frame.example.com/base/scheme", - "http://frame.example.com:8000/b", - "https://frame.example.com/c?a=b", - "http://example.com/sample/frame/relative", - "http://example.com/sample/", - "http://example.com/frame/absolute", - "ftp://frame.example.com/")); - } - - @Test - void shouldFindUrlsInIFrameElements() { - // Given - SpiderHtmlParser htmlParser = - new SpiderHtmlParser(new org.zaproxy.zap.spider.SpiderParam()); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage messageHtmlResponse = createMessageWith("IFrameElementsSpiderHtmlParser.html"); - Source source = createSource(messageHtmlResponse); - // When - boolean completelyParsed = - htmlParser.parseResource(messageHtmlResponse, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(7))); - assertThat( - listener.getUrlsFound(), - contains( - "http://iframe.example.com/base/scheme", - "http://iframe.example.com:8000/b", - "https://iframe.example.com/c?a=b", - "http://example.com/sample/iframe/relative", - "http://example.com/sample/", - "http://example.com/iframe/absolute", - "ftp://iframe.example.com/")); - } - - @Test - void shouldFindUrlsInIsIndexElements() { - // Given - SpiderHtmlParser htmlParser = - new SpiderHtmlParser(new org.zaproxy.zap.spider.SpiderParam()); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage messageHtmlResponse = createMessageWith("IsIndexElementsSpiderHtmlParser.html"); - Source source = createSource(messageHtmlResponse); - // When - boolean completelyParsed = - htmlParser.parseResource(messageHtmlResponse, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(3))); - assertThat( - listener.getUrlsFound(), - contains( - "http://example.com/sample/relative/action", - "http://example.com/absolute/action", - "https://isindex.example.com/action/target.html")); - } - - @Test - void shouldFindUrlsInLinkElements() { - // Given - SpiderHtmlParser htmlParser = - new SpiderHtmlParser(new org.zaproxy.zap.spider.SpiderParam()); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage messageHtmlResponse = createMessageWith("LinkElementsSpiderHtmlParser.html"); - Source source = createSource(messageHtmlResponse); - // When - boolean completelyParsed = - htmlParser.parseResource(messageHtmlResponse, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(7))); - assertThat( - listener.getUrlsFound(), - contains( - "http://link.example.com/base/scheme", - "http://link.example.com:8000/b", - "https://link.example.com/c?a=b", - "http://example.com/sample/link/relative", - "http://example.com/sample/", - "http://example.com/link/absolute", - "ftp://link.example.com/")); - } - - @Test - void shouldFindUrlsInInputElements() { - // Given - SpiderHtmlParser htmlParser = - new SpiderHtmlParser(new org.zaproxy.zap.spider.SpiderParam()); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage messageHtmlResponse = createMessageWith("InputElementsSpiderHtmlParser.html"); - Source source = createSource(messageHtmlResponse); - // When - boolean completelyParsed = - htmlParser.parseResource(messageHtmlResponse, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(3))); - assertThat( - listener.getUrlsFound(), - contains( - "http://example.com/sample/relative/src", - "http://example.com/absolute/src", - "https://input.example.com/external/inputsrc")); - } - - @Test - void shouldFindUrlsInObjectElements() { - // Given - SpiderHtmlParser htmlParser = - new SpiderHtmlParser(new org.zaproxy.zap.spider.SpiderParam()); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage messageHtmlResponse = createMessageWith("ObjectElementsSpiderHtmlParser.html"); - Source source = createSource(messageHtmlResponse); - // When - boolean completelyParsed = - htmlParser.parseResource(messageHtmlResponse, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(11))); - assertThat( - listener.getUrlsFound(), - contains( - "http://object.example.com/base/data", - "http://object.example.com:8000/data", - "https://object.example.com/data?a=b", - "http://example.com/sample/data/relative", - "http://example.com/sample/", - "http://example.com/data/absolute", - "ftp://object.example.com/data", - "http://object.example.com/codebase/scheme", - "https://object.example.com/codebase?a=b", - "http://example.com/sample/codebase/relative", - "http://example.com/codebase/absolute")); - } - - @Test - void shouldFindUrlsInScriptElements() { - // Given - SpiderHtmlParser htmlParser = - new SpiderHtmlParser(new org.zaproxy.zap.spider.SpiderParam()); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage messageHtmlResponse = createMessageWith("ScriptElementsSpiderHtmlParser.html"); - Source source = createSource(messageHtmlResponse); - // When - boolean completelyParsed = - htmlParser.parseResource(messageHtmlResponse, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(7))); - assertThat( - listener.getUrlsFound(), - contains( - "http://script.example.com/base/scheme", - "http://script.example.com:8000/b", - "https://script.example.com/c?a=b", - "http://example.com/sample/script/relative", - "http://example.com/sample/", - "http://example.com/script/absolute", - "ftp://script.example.com/")); - } - - @Test - void shouldFindUrlsInTableElements() { - // Given - SpiderHtmlParser htmlParser = - new SpiderHtmlParser(new org.zaproxy.zap.spider.SpiderParam()); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage messageHtmlResponse = createMessageWith("TableElementsSpiderHtmlParser.html"); - Source source = createSource(messageHtmlResponse); - // When - boolean completelyParsed = - htmlParser.parseResource(messageHtmlResponse, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(11))); - assertThat( - listener.getUrlsFound(), - contains( - "http://table.background.example.com/base/scheme", - "http://table.background.example.com:8000/b", - "https://table.background.example.com/c?a=b", - "http://example.com/sample/background/relative", - "http://example.com/sample/", - "http://example.com/background/absolute", - "ftp://background.example.com/", - "http://example.com/background/td_absolute1", - "http://example.com/background/td_absolute2", - "http://example.com/sample/background/td_relative1", - "http://td.background.example.com/")); - } - - @Test - void shouldFindUrlsInVideoElements() { - // Given - SpiderHtmlParser htmlParser = - new SpiderHtmlParser(new org.zaproxy.zap.spider.SpiderParam()); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage messageHtmlResponse = createMessageWith("VideoElementsSpiderHtmlParser.html"); - Source source = createSource(messageHtmlResponse); - // When - boolean completelyParsed = - htmlParser.parseResource(messageHtmlResponse, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(14))); - assertThat( - listener.getUrlsFound(), - contains( - "http://video.example.com/base/scheme", - "http://video.example.com:8000/b", - "https://video.example.com/c?a=b", - "http://example.com/sample/video/relative", - "http://example.com/sample/", - "http://example.com/video/absolute", - "ftp://video.example.com/", - "http://poster.example.com/", - "http://example.com/sample/poster/relative", - "http://example.com/media/cc0-videos/flower.webm", - "http://example.com/media/cc0-videos/flower.mp4", - "ftp://src.precedence.example.com/", - "http://example.com/media/cc0-videos/stillFound.webm", - "http://example.com/media/cc0-videos/stillFound.mp4")); - } - - @Test - void shouldFindUrlsInImgElements() { - // Given - SpiderHtmlParser htmlParser = - new SpiderHtmlParser(new org.zaproxy.zap.spider.SpiderParam()); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage messageHtmlResponse = createMessageWith("ImgElementsSpiderHtmlParser.html"); - Source source = createSource(messageHtmlResponse); - // When - boolean completelyParsed = - htmlParser.parseResource(messageHtmlResponse, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(24))); - assertThat( - listener.getUrlsFound(), - contains( - "http://img.example.com/base/scheme", - "http://img.example.com:8000/b", - "https://img.example.com/c?a=b", - "http://example.com/sample/img/relative", - "http://example.com/sample/", - "http://example.com/img/absolute", - "ftp://img.example.com/", - "http://example.com/sample/relative/longdesc", - "https://img.example.com/full/longdesc", - "http://example.com/img/lowsrc", - "https://video.example.com/dynsrc/video", - "http://example.com/test/html/body/img/srcset1x.found", - "http://example.com/test/html/body/img/srcset2x.found", - "http://example.com/test/html/body/img/normal_srcset.found", - "http://example.com/test/html/body/img/normal_srcset1.found", - "http://example.com/test/html/body/img/normal_srcset2.found", - "http://example.com/test/html/body/img/normal_srcset3.found", - "http://example.com/test/html/body/img/compact_srcset1.found", - "http://example.com/test/html/body/img/compact_srcset2.found", - "http://example.com/test/html/body/img/compact_srcset3.found", - "http://example.com/test/html/body/img/mixed_compact_srcset1.found", - "http://example.com/test/html/body/img/mixed_compact_srcset2.found", - "http://example.com/sample/pixel_width1.png", - "http://example.com/sample/pixel_width2.png")); - } - - @Test - void shouldFindUrlsInMetaElements() { - // Given - SpiderHtmlParser htmlParser = - new SpiderHtmlParser(new org.zaproxy.zap.spider.SpiderParam()); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage messageHtmlResponse = createMessageWith("MetaElementsSpiderHtmlParser.html"); - Source source = createSource(messageHtmlResponse); - // When - boolean completelyParsed = - htmlParser.parseResource(messageHtmlResponse, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(22))); - assertThat( - listener.getUrlsFound(), - contains( - "http://meta.example.com:8443/refresh/base/scheme", - "https://meta.example.com/refresh", - "http://example.com/sample/meta/refresh/relative", - "http://example.com/meta/refresh/absolute", - "http://meta.example.com/refresh/url/quoted/single", - "http://meta.example.com/refresh/url/quoted/double", - "ftp://meta.example.com/refresh", - "http://meta.example.com:8080/location/base/scheme", - "https://meta.example.com/location", - "http://example.com/sample/meta/location/relative", - "http://example.com/meta/location/absolute", - "ftp://meta.example.com/location", - "http://example.com/meta/csp", - "http://meta.example.com:4444/meta/base/csp/scheme", - "https://meta.example.com/meta/csp", - "http://example.com/sample/meta/csp/refresh/relative", - "ftp://meta.example.com/meta/csp/", - "http://example.com/meta/msapplication", - "http://meta.example.com:1337/meta/msapplication", - "https://meta.example.com/meta/msapplication", - "http://example.com/sample/meta/msapplication/refresh/relative", - "ftp://meta.example.com/meta/msapplication/refresh")); - } - - @Test - void shouldFindUrlsInString() { - // Given - SpiderHtmlParser htmlParser = - new SpiderHtmlParser(new org.zaproxy.zap.spider.SpiderParam()); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage messageHtmlResponse = createMessageWith("StringSpiderHtmlParser.html"); - Source source = createSource(messageHtmlResponse); - // When - boolean completelyParsed = - htmlParser.parseResource(messageHtmlResponse, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(3))); - assertThat( - listener.getUrlsFound(), - contains( - "http://example2.com/test/p/string/fullUrl", - "http://example.com/sample/with/base/tag", - "http://meta.example.com:8443/inline/string/scheme")); - } - - @Test - void shouldFindUrlsInCommentsWithElements() { - // AKA shouldNotFindPlainUrlsInCommentsWithElements - // Given - SpiderHtmlParser htmlParser = - new SpiderHtmlParser(new org.zaproxy.zap.spider.SpiderParam()); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage messageHtmlResponse = - createMessageWith("CommentWithElementsSpiderHtmlParser.html"); - Source source = createSource(messageHtmlResponse); - // When - boolean completelyParsed = - htmlParser.parseResource(messageHtmlResponse, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(9))); - assertThat( - listener.getUrlsFound(), - contains( - "http://a.example.com/", - "http://area.example.com/", - "http://frame.example.com/", - "http://iframe.example.com/", - "http://img.example.com/", - "http://link.example.com/", - "http://meta.example.com/refresh/", - "http://meta.example.com/location/", - "http://script.example.com/")); - } - - @Test - void shouldNotFindUrlsInCommentsWithElementsIfNotEnabledToParseComments() { - // Given - org.zaproxy.zap.spider.SpiderParam spiderOptions = createSpiderParamWithConfig(); - spiderOptions.setParseComments(false); - SpiderHtmlParser htmlParser = new SpiderHtmlParser(spiderOptions); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage messageHtmlResponse = - createMessageWith("CommentWithElementsSpiderHtmlParser.html"); - Source source = createSource(messageHtmlResponse); - // When - boolean completelyParsed = - htmlParser.parseResource(messageHtmlResponse, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(0))); - assertThat(listener.getUrlsFound(), is(empty())); - } - - @Test - void shouldFindUrlsInCommentsWithoutElements() { - // Given - SpiderHtmlParser htmlParser = - new SpiderHtmlParser(new org.zaproxy.zap.spider.SpiderParam()); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage messageHtmlResponse = - createMessageWith("CommentWithoutElementsSpiderHtmlParser.html"); - Source source = createSource(messageHtmlResponse); - // When - boolean completelyParsed = - htmlParser.parseResource(messageHtmlResponse, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(10))); - assertThat( - listener.getUrlsFound(), - contains( - "http://plaincomment.example.com/", - "http://plaincomment.example.com/z.php?x=y", - "http://plaincomment.example.com/c.pl?x=y", - "https://plaincomment.example.com/d.asp?x=y", - "https://plaincomment.example.com/e/e1/e2.html?x=y", - "https://plaincomment.example.com/surrounded/with/parenthesis", - "https://plaincomment.example.com/surrounded/with/brackets", - "https://plaincomment.example.com/surrounded/with/curly/brackets", - "http://plaincomment.example.com/variant1", - "http://plaincomment.example.com/variant2")); - } - - @Test - void shouldNotFindUrlsInCommentsWithoutElementsIfNotEnabledToParseComments() { - // Given - org.zaproxy.zap.spider.SpiderParam spiderOptions = createSpiderParamWithConfig(); - spiderOptions.setParseComments(false); - SpiderHtmlParser htmlParser = new SpiderHtmlParser(spiderOptions); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage messageHtmlResponse = - createMessageWith("CommentWithoutElementsSpiderHtmlParser.html"); - Source source = createSource(messageHtmlResponse); - // When - boolean completelyParsed = - htmlParser.parseResource(messageHtmlResponse, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(0))); - assertThat(listener.getUrlsFound(), is(empty())); - } - - @ParameterizedTest - @ValueSource( - strings = { - "DocTypeWithFullUrl.html", - "DocTypeWithRelativeUrl.html", - "ManifestWithFullUrl.html", - "ManifestWithRelativeUrl.html", - "BackgroundWithFullUrl.html", - "BackgroundWithRelativeUrl.html" - }) - void shouldFindUrlInFile(String file) { - // Given - org.zaproxy.zap.spider.SpiderParam spiderOptions = createSpiderParamWithConfig(); - spiderOptions.setParseComments(false); - SpiderHtmlParser htmlParser = new SpiderHtmlParser(spiderOptions); - TestSpiderParserListener listener = createTestSpiderParserListener(); - htmlParser.addSpiderParserListener(listener); - HttpMessage messageHtmlResponse = createMessageWith(file); - Source source = createSource(messageHtmlResponse); - // When - boolean completelyParsed = - htmlParser.parseResource(messageHtmlResponse, source, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - assertThat(listener.getNumberOfUrlsFound(), is(equalTo(1))); - assertThat(listener.getUrlsFound(), contains("http://example.com/found")); - } - - private static HttpMessage createMessageWith(String filename) { - return createMessageWith(filename, "/"); - } - - private static HttpMessage createMessageWith(String filename, String requestUri) { - HttpMessage message = new HttpMessage(); - try { - String fileContents = readFile(BASE_DIR_HTML_FILES.resolve(filename)); - message.setRequestHeader("GET " + requestUri + " HTTP/1.1\r\nHost: example.com\r\n"); - message.setResponseHeader( - "HTTP/1.1 200 OK\r\n" - + "Content-Type: text/html; charset=UTF-8\r\n" - + "Content-Length: " - + fileContents.length()); - message.setResponseBody(fileContents); - } catch (Exception e) { - throw new RuntimeException(e); - } - return message; - } -} diff --git a/zap/src/test/java/org/zaproxy/zap/spider/parser/SpiderHttpHeaderParserUnitTest.java b/zap/src/test/java/org/zaproxy/zap/spider/parser/SpiderHttpHeaderParserUnitTest.java deleted file mode 100644 index 81933d88516..00000000000 --- a/zap/src/test/java/org/zaproxy/zap/spider/parser/SpiderHttpHeaderParserUnitTest.java +++ /dev/null @@ -1,215 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2022 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.spider.parser; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.contains; -import static org.hamcrest.Matchers.empty; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.Mockito.mock; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; -import org.parosproxy.paros.network.HttpHeader; -import org.parosproxy.paros.network.HttpMalformedHeaderException; -import org.parosproxy.paros.network.HttpMessage; - -/** Unit test for {@link SpiderHttpHeaderParser}. */ -@SuppressWarnings("deprecation") -class SpiderHttpHeaderParserUnitTest extends SpiderParserTestUtils { - - private static final String ROOT_PATH = "/"; - private static final int BASE_DEPTH = 0; - - private org.zaproxy.zap.spider.parser.SpiderHttpHeaderParser headerParser; - - @BeforeEach - void setup() { - org.zaproxy.zap.spider.SpiderParam spiderOptions = - mock(org.zaproxy.zap.spider.SpiderParam.class); - headerParser = new org.zaproxy.zap.spider.parser.SpiderHttpHeaderParser(spiderOptions); - } - - @Test - void shouldParseAnyMessage() { - // Given - HttpMessage msg = createMessage(); - // When - boolean canParse = headerParser.canParseResource(msg, ROOT_PATH, false); - // Then - assertThat(canParse, is(equalTo(true))); - } - - @Test - void shouldParseAnyMessageEvenIfAlreadyParsed() { - // Given - boolean alreadyParsed = true; - HttpMessage msg = createMessage(); - // When - boolean canParse = headerParser.canParseResource(msg, ROOT_PATH, alreadyParsed); - // Then - assertThat(canParse, is(equalTo(true))); - } - - @Test - void shouldFailToParseAnUndefinedMessage() { - // Given - HttpMessage undefinedMessage = null; - // When / Then - assertThrows( - NullPointerException.class, - () -> headerParser.parseResource(undefinedMessage, null, BASE_DEPTH)); - } - - @Test - void shouldNotExtractUrlIfNoUrlHeadersPresent() { - // Given - HttpMessage msg = createMessage(); - TestSpiderParserListener listener = createAndAddTestSpiderParserListener(headerParser); - // When - boolean parsed = headerParser.parseResource(msg, null, BASE_DEPTH); - // Then - assertThat(parsed, is(equalTo(false))); - assertThat(listener.getUrlsFound(), is(empty())); - } - - @ParameterizedTest - @ValueSource(strings = {HttpHeader.CONTENT_LOCATION, HttpHeader.REFRESH, HttpHeader.LINK}) - void shouldNotExtractUrlIfUrlHeaderIsEmpty(String header) { - // Given - HttpMessage msg = createMessage(); - msg.getResponseHeader().addHeader(header, ""); - TestSpiderParserListener listener = createAndAddTestSpiderParserListener(headerParser); - // When - boolean parsed = headerParser.parseResource(msg, null, 0); - // Then - assertThat(parsed, is(equalTo(false))); - assertThat(listener.getUrlsFound(), is(empty())); - } - - @Test - void shouldExtractUrlFromContentLocationHeader() { - // Given - String value = "http://example.com/contentlocation"; - HttpMessage msg = createMessage(); - msg.getResponseHeader().addHeader(HttpHeader.CONTENT_LOCATION, value); - TestSpiderParserListener listener = createAndAddTestSpiderParserListener(headerParser); - // When - boolean parsed = headerParser.parseResource(msg, null, BASE_DEPTH); - // Then - assertThat(parsed, is(equalTo(false))); - assertThat(listener.getUrlsFound(), contains(value)); - } - - @Test - void shouldExtractRelativeUrlFromContentLocationHeader() { - // Given - String url = "/rel/redirection"; - HttpMessage msg = createMessage(); - msg.getResponseHeader().addHeader(HttpHeader.CONTENT_LOCATION, url); - TestSpiderParserListener listener = createAndAddTestSpiderParserListener(headerParser); - // When - boolean parsed = headerParser.parseResource(msg, null, BASE_DEPTH); - // Then - assertThat(parsed, is(equalTo(false))); - assertThat(listener.getUrlsFound(), contains("http://example.com" + url)); - } - - @Test - void shouldExtractUrlsFromLinkHeader() { - // Given - String url1 = "http://example.com/link1"; - String url2 = "/link2"; - HttpMessage msg = createMessage(); - msg.getResponseHeader() - .addHeader( - HttpHeader.LINK, - "<" + url1 + ">; param1=value1; param2=\"value2\";<" + url2 + ">"); - TestSpiderParserListener listener = createAndAddTestSpiderParserListener(headerParser); - // When - boolean parsed = headerParser.parseResource(msg, null, BASE_DEPTH); - // Then - assertThat(parsed, is(equalTo(false))); - assertThat(listener.getUrlsFound(), contains(url1, "http://example.com" + url2)); - } - - @ParameterizedTest - @ValueSource( - strings = { - "", - "more>bad resources; - private final List urls; - - private TestSpiderParserListener() { - resources = new ArrayList<>(); - urls = new ArrayList<>(); - } - - int getNumberOfUrlsFound() { - return urls.size(); - } - - List getUrlsFound() { - return urls; - } - - int getNumberOfResourcesFound() { - return resources.size(); - } - - List getResourcesFound() { - return resources; - } - - @Override - public void resourceFound(SpiderResourceFound resourceFound) { - urls.add(resourceFound.getUri()); - resources.add(resourceFound); - } - - boolean isResourceFound() { - return false; - } - } - - static SpiderResourceFound uriResource(HttpMessage message, int depth, String uri) { - return SpiderResourceFound.builder() - .setMessage(message) - .setDepth(depth) - .setUri(uri) - .build(); - } - - static SpiderResourceFound uriResource( - HttpMessage message, int depth, String uri, boolean shouldIgnore) { - return uriResource(message, depth, uri, shouldIgnore, new ArrayList<>()); - } - - static SpiderResourceFound uriResource( - HttpMessage message, - int depth, - String uri, - boolean shouldIgnore, - List requestHeaders) { - return SpiderResourceFound.builder() - .setMessage(message) - .setDepth(depth) - .setUri(uri) - .setShouldIgnore(shouldIgnore) - .setHeaders(requestHeaders) - .build(); - } - - static SpiderResourceFound postResource( - HttpMessage message, int depth, String uri, String requestBody) { - return postResource(message, depth, uri, requestBody, new ArrayList<>()); - } - - static SpiderResourceFound postResource( - HttpMessage message, - int depth, - String uri, - String requestBody, - List requestHeaders) { - return SpiderResourceFound.builder() - .setMessage(message) - .setDepth(depth) - .setUri(uri) - .setMethod(HttpRequestHeader.POST) - .setBody(requestBody) - .setHeaders(requestHeaders) - .build(); - } - - static String params(String... params) { - if (params == null || params.length == 0) { - return ""; - } - - StringBuilder strBuilder = new StringBuilder(); - for (String param : params) { - if (strBuilder.length() > 0) { - strBuilder.append('&'); - } - strBuilder.append(param); - } - return strBuilder.toString(); - } - - static String param(String name, String value) { - try { - return URLEncoder.encode(name, StandardCharsets.UTF_8.name()) - + "=" - + URLEncoder.encode(value, StandardCharsets.UTF_8.name()); - } catch (UnsupportedEncodingException e) { - throw new RuntimeException(e); - } - } -} diff --git a/zap/src/test/java/org/zaproxy/zap/spider/parser/SpiderParserUnitTest.java b/zap/src/test/java/org/zaproxy/zap/spider/parser/SpiderParserUnitTest.java deleted file mode 100644 index c12bd785055..00000000000 --- a/zap/src/test/java/org/zaproxy/zap/spider/parser/SpiderParserUnitTest.java +++ /dev/null @@ -1,172 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2021 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.spider.parser; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.contains; -import static org.hamcrest.Matchers.empty; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.not; -import static org.hamcrest.Matchers.nullValue; -import static org.mockito.Mockito.mock; - -import net.htmlparser.jericho.Source; -import org.junit.jupiter.api.Test; -import org.parosproxy.paros.network.HttpMessage; - -/** Unit test for {@link SpiderParser}. */ -@SuppressWarnings("deprecation") -class SpiderParserUnitTest extends SpiderParserTestUtils { - - @Test - void shouldHaveNonNullLogger() { - // Given - TestSpiderParser testSpiderParser = new TestSpiderParser(); - // When / Then - assertThat(testSpiderParser.getLogger(), is(not(nullValue()))); - } - - @Test - void shouldNotifyListenersOfResourceFound() { - // Given - TestSpiderParser testSpiderParser = new TestSpiderParser(); - TestSpiderParserListener listener1 = createTestSpiderParserListener(); - TestSpiderParserListener listener2 = createTestSpiderParserListener(); - org.zaproxy.zap.spider.parser.SpiderResourceFound resourceFound1 = - mock(org.zaproxy.zap.spider.parser.SpiderResourceFound.class); - org.zaproxy.zap.spider.parser.SpiderResourceFound resourceFound2 = - mock(org.zaproxy.zap.spider.parser.SpiderResourceFound.class); - // When - testSpiderParser.addSpiderParserListener(listener1); - testSpiderParser.addSpiderParserListener(listener2); - testSpiderParser.notifyListenersResourceFound(resourceFound1); - testSpiderParser.notifyListenersResourceFound(resourceFound2); - // Then - assertThat(listener1.getResourcesFound(), contains(resourceFound1, resourceFound2)); - assertThat(listener2.getResourcesFound(), contains(resourceFound1, resourceFound2)); - } - - @Test - void shouldNotNotifyRemovedListenerOfResourceFound() { - // Given - TestSpiderParser testSpiderParser = new TestSpiderParser(); - TestSpiderParserListener listener1 = createTestSpiderParserListener(); - testSpiderParser.addSpiderParserListener(listener1); - TestSpiderParserListener listener2 = createTestSpiderParserListener(); - testSpiderParser.addSpiderParserListener(listener2); - org.zaproxy.zap.spider.parser.SpiderResourceFound resourceFound1 = - mock(org.zaproxy.zap.spider.parser.SpiderResourceFound.class); - org.zaproxy.zap.spider.parser.SpiderResourceFound resourceFound2 = - mock(org.zaproxy.zap.spider.parser.SpiderResourceFound.class); - // When - testSpiderParser.notifyListenersResourceFound(resourceFound1); - testSpiderParser.removeSpiderParserListener(listener2); - testSpiderParser.notifyListenersResourceFound(resourceFound2); - // Then - assertThat(listener1.getResourcesFound(), contains(resourceFound1, resourceFound2)); - assertThat(listener2.getResourcesFound(), contains(resourceFound1)); - } - - @Test - @SuppressWarnings("deprecation") - void shouldNotifyListenersOfUriResourceFound() { - // Given - TestSpiderParser testSpiderParser = new TestSpiderParser(); - TestSpiderParserListener listener = createTestSpiderParserListener(); - testSpiderParser.addSpiderParserListener(listener); - HttpMessage message = mock(HttpMessage.class); - int depth = 42; - String uri = "https://example.com"; - // When - testSpiderParser.notifyListenersResourceFound(message, depth, uri); - // Then - assertThat(listener.getResourcesFound(), contains(uriResource(message, depth, uri))); - } - - @Test - @SuppressWarnings("deprecation") - void shouldNotifyListenersOfPostResourceFound() { - // Given - TestSpiderParser testSpiderParser = new TestSpiderParser(); - TestSpiderParserListener listener = createTestSpiderParserListener(); - testSpiderParser.addSpiderParserListener(listener); - HttpMessage message = mock(HttpMessage.class); - int depth = 42; - String uri = "https://example.com"; - String body = "body"; - // When - testSpiderParser.notifyListenersPostResourceFound(message, depth, uri, body); - // Then - assertThat(listener.getResourcesFound(), contains(postResource(message, depth, uri, body))); - } - - @Test - void shouldNotifyListenersOfProcessedUrl() { - // Given - TestSpiderParser testSpiderParser = new TestSpiderParser(); - TestSpiderParserListener listener = createTestSpiderParserListener(); - testSpiderParser.addSpiderParserListener(listener); - HttpMessage message = mock(HttpMessage.class); - int depth = 42; - String baseUrl = "https://example.com/"; - String localUrl = "/path/"; - String expectedUri = "https://example.com/path/"; - // When - testSpiderParser.processURL(message, depth, localUrl, baseUrl); - // Then - assertThat( - listener.getResourcesFound(), - contains(uriResource(message, depth + 1, expectedUri))); - } - - @Test - void shouldNotNotifyListenersOfMalformedProcessedUrl() { - // Given - TestSpiderParser testSpiderParser = new TestSpiderParser(); - TestSpiderParserListener listener = createTestSpiderParserListener(); - testSpiderParser.addSpiderParserListener(listener); - HttpMessage message = mock(HttpMessage.class); - int depth = 42; - String baseUrl = "/"; - String localUrl = "/"; - // When - testSpiderParser.processURL(message, depth, localUrl, baseUrl); - // Then - assertThat(listener.getResourcesFound(), is(empty())); - } - - private static class TestSpiderParser extends SpiderParser { - - TestSpiderParser() { - super(mock(org.zaproxy.zap.spider.SpiderParam.class)); - } - - @Override - public boolean parseResource(HttpMessage message, Source source, int depth) { - return true; - } - - @Override - public boolean canParseResource( - HttpMessage message, String path, boolean wasAlreadyConsumed) { - return true; - } - } -} diff --git a/zap/src/test/java/org/zaproxy/zap/spider/parser/SpiderRedirectParserUnitTest.java b/zap/src/test/java/org/zaproxy/zap/spider/parser/SpiderRedirectParserUnitTest.java deleted file mode 100644 index 05f3b5c8ee8..00000000000 --- a/zap/src/test/java/org/zaproxy/zap/spider/parser/SpiderRedirectParserUnitTest.java +++ /dev/null @@ -1,196 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2017 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.spider.parser; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.contains; -import static org.hamcrest.Matchers.empty; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.Mockito.mock; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.parosproxy.paros.network.HttpHeader; -import org.parosproxy.paros.network.HttpMalformedHeaderException; -import org.parosproxy.paros.network.HttpMessage; -import org.parosproxy.paros.network.HttpStatusCode; - -/** Unit test for {@link SpiderRedirectParser}. */ -@SuppressWarnings("deprecation") -class SpiderRedirectParserUnitTest extends SpiderParserTestUtils { - - private static final String ROOT_PATH = "/"; - private static final int BASE_DEPTH = 0; - - private static final List NON_REDIRECTION_STATUS_CODES; - private static final List REDIRECTION_STATUS_CODES; - - static { - NON_REDIRECTION_STATUS_CODES = new ArrayList<>(); - REDIRECTION_STATUS_CODES = new ArrayList<>(); - Arrays.stream(HttpStatusCode.CODES) - .forEach( - code -> { - if (HttpStatusCode.isRedirection(code)) { - REDIRECTION_STATUS_CODES.add(code); - } else { - NON_REDIRECTION_STATUS_CODES.add(code); - } - }); - } - - private SpiderRedirectParser redirectParser; - - @BeforeEach - void setup() { - org.zaproxy.zap.spider.SpiderParam spiderOptions = - mock(org.zaproxy.zap.spider.SpiderParam.class); - redirectParser = new SpiderRedirectParser(spiderOptions); - } - - @Test - void shouldFailToEvaluateAnUndefinedMessage() { - // Given - HttpMessage undefinedMessage = null; - // When / Then - assertThrows( - NullPointerException.class, - () -> redirectParser.canParseResource(undefinedMessage, ROOT_PATH, false)); - } - - @Test - void shouldNotParseNonRedirectionMessages() { - // Given - for (int statusCode : NON_REDIRECTION_STATUS_CODES) { - HttpMessage msg = createMessageWithStatusCode(statusCode); - // When - boolean canParse = redirectParser.canParseResource(msg, ROOT_PATH, false); - // Then - assertThat(Integer.toString(statusCode), canParse, is(equalTo(false))); - } - } - - @Test - void shouldParseRedirectionMessages() { - // Given - for (int statusCode : REDIRECTION_STATUS_CODES) { - HttpMessage msg = createMessageWithStatusCode(statusCode); - // When - boolean canParse = redirectParser.canParseResource(msg, ROOT_PATH, false); - // Then - assertThat(Integer.toString(statusCode), canParse, is(equalTo(true))); - } - } - - @Test - void shouldParseRedirectionMessageEvenIfAlreadyParsed() { - // Given - boolean alreadyParsed = true; - HttpMessage msg = createMessageWithStatusCode(HttpStatusCode.FOUND); - // When - boolean canParse = redirectParser.canParseResource(msg, ROOT_PATH, alreadyParsed); - // Then - assertThat(canParse, is(equalTo(true))); - } - - @Test - void shouldFailToParseAnUndefinedMessage() { - // Given - HttpMessage undefinedMessage = null; - // When / Then - assertThrows( - NullPointerException.class, - () -> redirectParser.parseResource(undefinedMessage, null, BASE_DEPTH)); - } - - @Test - void shouldExtractUrlFromLocationHeader() { - // Given - String location = "http://example.com/redirection"; - HttpMessage msg = createMessageWithLocationAndStatusCode(location, HttpStatusCode.FOUND); - TestSpiderParserListener listener = createAndAddTestSpiderParserListener(redirectParser); - // When - boolean parsed = redirectParser.parseResource(msg, null, BASE_DEPTH); - // Then - assertThat(parsed, is(equalTo(true))); - assertThat(listener.getUrlsFound(), contains(location)); - } - - @Test - void shouldExtractRelativeUrlFromLocationHeader() { - // Given - String location = "/rel/redirection"; - HttpMessage msg = createMessageWithLocationAndStatusCode(location, HttpStatusCode.FOUND); - TestSpiderParserListener listener = createAndAddTestSpiderParserListener(redirectParser); - // When - boolean parsed = redirectParser.parseResource(msg, null, BASE_DEPTH); - // Then - assertThat(parsed, is(equalTo(true))); - assertThat(listener.getUrlsFound(), contains("http://example.com" + location)); - } - - @Test - void shouldNotExtractUrlIfLocationHeaderIsEmpty() { - // Given - String location = ""; - HttpMessage msg = createMessageWithLocationAndStatusCode(location, HttpStatusCode.FOUND); - TestSpiderParserListener listener = createAndAddTestSpiderParserListener(redirectParser); - // When - boolean parsed = redirectParser.parseResource(msg, null, 0); - // Then - assertThat(parsed, is(equalTo(true))); - assertThat(listener.getUrlsFound(), is(empty())); - } - - @Test - void shouldNotExtractUrlIfLocationHeaderIsNotPresent() { - // Given - HttpMessage msg = createMessageWithStatusCode(HttpStatusCode.FOUND); - TestSpiderParserListener listener = createAndAddTestSpiderParserListener(redirectParser); - // When - boolean parsed = redirectParser.parseResource(msg, null, BASE_DEPTH); - // Then - assertThat(parsed, is(equalTo(true))); - assertThat(listener.getUrlsFound(), is(empty())); - } - - private static HttpMessage createMessageWithStatusCode(int statusCode) { - HttpMessage msg = new HttpMessage(); - try { - msg.setRequestHeader("GET / HTTP/1.1\r\nHost: example.com\r\n"); - msg.setResponseHeader("HTTP/1.1 " + statusCode + " Reason\r\n"); - } catch (HttpMalformedHeaderException e) { - throw new RuntimeException(e); - } - return msg; - } - - private static HttpMessage createMessageWithLocationAndStatusCode( - String location, int statusCode) { - HttpMessage msg = createMessageWithStatusCode(statusCode); - msg.getResponseHeader().addHeader(HttpHeader.LOCATION, location); - return msg; - } -} diff --git a/zap/src/test/java/org/zaproxy/zap/spider/parser/SpiderResourceFoundUnitTest.java b/zap/src/test/java/org/zaproxy/zap/spider/parser/SpiderResourceFoundUnitTest.java deleted file mode 100644 index 7d585d06813..00000000000 --- a/zap/src/test/java/org/zaproxy/zap/spider/parser/SpiderResourceFoundUnitTest.java +++ /dev/null @@ -1,210 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2021 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.spider.parser; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.contains; -import static org.hamcrest.Matchers.empty; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.params.provider.Arguments.arguments; -import static org.mockito.Mockito.mock; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.stream.Stream; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; -import org.parosproxy.paros.network.HttpHeaderField; -import org.parosproxy.paros.network.HttpMessage; - -/** Unit test for {@link SpiderResourceFound}. */ -@SuppressWarnings("deprecation") -class SpiderResourceFoundUnitTest { - - private SpiderResourceFound.Builder builder; - - @BeforeEach - void setUp() { - builder = SpiderResourceFound.builder(); - } - - @Test - void shouldSetMessage() { - // Given - HttpMessage message = mock(HttpMessage.class); - // When - SpiderResourceFound resource = builder.setMessage(message).build(); - // Then - assertThat(resource.getMessage(), is(equalTo(message))); - } - - @Test - void shouldSetShouldIgnore() { - // Given - boolean shouldIgnore = true; - // When - SpiderResourceFound resource = builder.setShouldIgnore(shouldIgnore).build(); - // Then - assertThat(resource.isShouldIgnore(), is(equalTo(shouldIgnore))); - } - - @Test - void shouldSetUri() { - // Given - String uri = "uri"; - // When - SpiderResourceFound resource = builder.setUri(uri).build(); - // Then - assertThat(resource.getUri(), is(equalTo(uri))); - } - - @Test - void shouldThrowExceptionForNullUri() { - // Given - String uri = null; - // When / Then - assertThrows(IllegalArgumentException.class, () -> builder.setUri(uri)); - } - - @Test - void shouldSetDepth() { - // Given - int depth = 42; - // When - SpiderResourceFound resource = builder.setDepth(depth).build(); - // Then - assertThat(resource.getDepth(), is(equalTo(depth))); - } - - @Test - void shouldThrowExceptionForNegativeDepth() { - // Given - int depth = -99; - // When / Then - assertThrows(IllegalArgumentException.class, () -> builder.setDepth(depth)); - } - - @Test - void shouldSetMethod() { - // Given - String method = "method"; - // When - SpiderResourceFound resource = builder.setMethod(method).build(); - // Then - assertThat(resource.getMethod(), is(equalTo(method))); - } - - @Test - void shouldThrowExceptionForNullMethod() { - // Given - String method = null; - // When / Then - assertThrows(IllegalArgumentException.class, () -> builder.setMethod(method)); - } - - @Test - void shouldSetBody() { - // Given - String body = "body"; - // When - SpiderResourceFound resource = builder.setBody(body).build(); - // Then - assertThat(resource.getBody(), is(equalTo(body))); - } - - @Test - void shouldThrowExceptionForNullBody() { - // Given - String body = null; - // When / Then - assertThrows(IllegalArgumentException.class, () -> builder.setBody(body)); - } - - @Test - void shouldThrowExceptionForNullHeaders() { - // Given - List headers = null; - // When / Then - assertThrows(IllegalArgumentException.class, () -> builder.setHeaders(headers)); - } - - @Test - void shouldThrowExceptionForNullHeaderField() { - // Given - List headers = new ArrayList<>(); - headers.add(null); - // When / Then - assertThrows(IllegalArgumentException.class, () -> builder.setHeaders(headers)); - } - - @Test - void shouldSetHeaders() { - // Given - HttpHeaderField header1 = new HttpHeaderField("name 1", "value 1"); - HttpHeaderField header2 = new HttpHeaderField("name 2", "value 2"); - List headers = Arrays.asList(header1, header2); - // When - SpiderResourceFound resource = builder.setHeaders(headers).build(); - // Then - assertThat(resource.getHeaders(), contains(header1, header2)); - } - - @Test - void shouldSetEmptyHeaders() { - // Given - HttpHeaderField header1 = new HttpHeaderField("name 1", "value 1"); - HttpHeaderField header2 = new HttpHeaderField("name 2", "value 2"); - List headers = Arrays.asList(header1, header2); - builder.setHeaders(headers); - // When - SpiderResourceFound resource = builder.setHeaders(Collections.emptyList()).build(); - // Then - assertThat(resource.getHeaders(), is(empty())); - } - - static Stream invalidHeadersProvider() { - return Stream.of( - arguments(null, "123"), - arguments("", "123"), - arguments(" ", "123"), - arguments("name", null)); - } - - @ParameterizedTest - @MethodSource("invalidHeadersProvider") - void shouldSkipInvalidHeaders(String name, String value) { - // Given - HttpHeaderField header1 = new HttpHeaderField("name 1", "value 1"); - HttpHeaderField invalidHeader = new HttpHeaderField(name, value); - HttpHeaderField header2 = new HttpHeaderField("name 2", "value 2"); - List headers = Arrays.asList(header1, invalidHeader, header2); - // When - SpiderResourceFound resource = builder.setHeaders(headers).build(); - // Then - assertThat(resource.getHeaders(), contains(header1, header2)); - } -} diff --git a/zap/src/test/java/org/zaproxy/zap/spider/parser/SpiderRobotstxtParserUnitTest.java b/zap/src/test/java/org/zaproxy/zap/spider/parser/SpiderRobotstxtParserUnitTest.java deleted file mode 100644 index 5d8e10c8a36..00000000000 --- a/zap/src/test/java/org/zaproxy/zap/spider/parser/SpiderRobotstxtParserUnitTest.java +++ /dev/null @@ -1,223 +0,0 @@ -/* - * Zed Attack Proxy (ZAP) and its related class files. - * - * ZAP is an HTTP/HTTPS proxy for assessing web application security. - * - * Copyright 2018 The ZAP Development Team - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.zaproxy.zap.spider.parser; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.contains; -import static org.hamcrest.Matchers.empty; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; -import static org.junit.jupiter.api.Assertions.assertThrows; - -import org.junit.jupiter.api.Test; -import org.parosproxy.paros.network.HttpMessage; - -/** Unit test for {@link SpiderRobotstxtParser}. */ -@SuppressWarnings("deprecation") -class SpiderRobotstxtParserUnitTest extends SpiderParserTestUtils { - - private static final String ROOT_PATH = "/"; - private static final String ROBOTS_TXT_PATH = "/robots.txt"; - private static final int BASE_DEPTH = 0; - - @Test - void shouldRequireNonNullSpiderParam() { - // Given - org.zaproxy.zap.spider.SpiderParam spiderParam = null; - // When / Then - assertThrows(NullPointerException.class, () -> new SpiderRobotstxtParser(spiderParam)); - } - - @Test - void shouldNotFailToEvaluateAnUndefinedPath() { - // Given - String path = null; - SpiderRobotstxtParser spiderParser = - new SpiderRobotstxtParser(new org.zaproxy.zap.spider.SpiderParam()); - // When / Then - assertDoesNotThrow(() -> spiderParser.canParseResource(null, path, false)); - } - - @Test - void shouldParseRobotsTxtPath() { - // Given - SpiderRobotstxtParser spiderParser = - new SpiderRobotstxtParser(new org.zaproxy.zap.spider.SpiderParam()); - // When - boolean canParse = spiderParser.canParseResource(null, ROBOTS_TXT_PATH, false); - // Then - assertThat(canParse, is(equalTo(true))); - } - - @Test - void shouldParseRobotsTxtPathWithDifferentCase() { - // Given - SpiderRobotstxtParser spiderParser = - new SpiderRobotstxtParser(new org.zaproxy.zap.spider.SpiderParam()); - // When - boolean canParse = spiderParser.canParseResource(null, "/RoBoTs.TxT", false); - // Then - assertThat(canParse, is(equalTo(true))); - } - - @Test - void shouldParseRobotsTxtPathEvenIfAlreadyParsed() { - // Given - SpiderRobotstxtParser spiderParser = - new SpiderRobotstxtParser(new org.zaproxy.zap.spider.SpiderParam()); - boolean parsed = true; - // When - boolean canParse = spiderParser.canParseResource(null, ROBOTS_TXT_PATH, parsed); - // Then - assertThat(canParse, is(equalTo(true))); - } - - @Test - void shouldNotParseNonRobotsTxtPath() { - // Given - SpiderRobotstxtParser spiderParser = - new SpiderRobotstxtParser(new org.zaproxy.zap.spider.SpiderParam()); - // When - boolean canParse = spiderParser.canParseResource(null, ROOT_PATH, false); - // Then - assertThat(canParse, is(equalTo(false))); - } - - @Test - void shouldFailToParseAnUndefinedMessage() { - // Given - HttpMessage undefinedMessage = null; - SpiderRobotstxtParser spiderParser = - new SpiderRobotstxtParser(new org.zaproxy.zap.spider.SpiderParam()); - // When / Then - assertThrows( - NullPointerException.class, - () -> spiderParser.parseResource(undefinedMessage, null, BASE_DEPTH)); - } - - @Test - void shouldNotBeCompletelyParsedIfParseDisabled() { - // Given - org.zaproxy.zap.spider.SpiderParam spiderParam = createSpiderParamWithConfig(); - spiderParam.setParseRobotsTxt(false); - SpiderRobotstxtParser spiderParser = new SpiderRobotstxtParser(spiderParam); - HttpMessage message = createMessageWith(""); - // When - boolean completelyParsed = spiderParser.parseResource(message, null, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(false))); - } - - @Test - void shouldBeAlwaysCompletelyParsedIfParseEnabled() { - // Given - SpiderRobotstxtParser spiderParser = - new SpiderRobotstxtParser(new org.zaproxy.zap.spider.SpiderParam()); - HttpMessage message = createMessageWith(""); - // When - boolean completelyParsed = spiderParser.parseResource(message, null, BASE_DEPTH); - // Then - assertThat(completelyParsed, is(equalTo(true))); - } - - @Test - void shouldNotFindUrlsIfThereIsNone() { - // Given - SpiderRobotstxtParser spiderParser = - new SpiderRobotstxtParser(new org.zaproxy.zap.spider.SpiderParam()); - TestSpiderParserListener listener = createTestSpiderParserListener(); - spiderParser.addSpiderParserListener(listener); - HttpMessage message = - createMessageWith( - body( - "# Just Comments & User-Agents...", - "User-Agent: *", - "# Disallow: /x/y/z", - "User-Agent: bot", - "

",
-                                "# Allow: /a/b/c",
-                                "",
-                                "# ...",
-                                "Allow:   # no path"));
-        // When
-        spiderParser.parseResource(message, null, BASE_DEPTH);
-        // Then
-        assertThat(listener.getUrlsFound(), is(empty()));
-    }
-
-    @Test
-    void shouldFindUrls() {
-        // Given
-        SpiderRobotstxtParser spiderParser =
-                new SpiderRobotstxtParser(new org.zaproxy.zap.spider.SpiderParam());
-        TestSpiderParserListener listener = createTestSpiderParserListener();
-        spiderParser.addSpiderParserListener(listener);
-        HttpMessage messageHtmlResponse =
-                createMessageWith(
-                        body(
-                                "User-Agent: *",
-                                "Disallow: /x/y/z    # Comment",
-                                " User-Agent: bot     # Comment",
-                                "Allow: /a/b/c.html",
-                                "
 Allow: /nohtmltags/",
-                                "  Allow:    /%  ",
-                                "Allow: /%20file.txt",
-                                "Allow: /abc/*"));
-        // When
-        spiderParser.parseResource(messageHtmlResponse, null, BASE_DEPTH);
-        // Then
-        assertThat(
-                listener.getUrlsFound(),
-                contains(
-                        "http://example.com/x/y/z",
-                        "http://example.com/a/b/c.html",
-                        "http://example.com/nohtmltags/",
-                        "http://example.com/%25",
-                        "http://example.com/%20file.txt",
-                        "http://example.com/abc/"));
-    }
-
-    private static HttpMessage createMessageWith(String body) {
-        HttpMessage message = new HttpMessage();
-        try {
-            message.setRequestHeader("GET / HTTP/1.1\r\nHost: example.com\r\n");
-            message.setResponseHeader("HTTP/1.1 200 OK\r\nContent-Length: " + body.length());
-            message.setResponseBody(body);
-        } catch (Exception e) {
-            throw new RuntimeException(e);
-        }
-        return message;
-    }
-
-    private static String body(String... strings) {
-        if (strings == null || strings.length == 0) {
-            return "";
-        }
-        StringBuilder strBuilder = new StringBuilder(strings.length * 25);
-        for (String string : strings) {
-            if (strBuilder.length() > 0) {
-                strBuilder.append("\n");
-            }
-            strBuilder.append(string);
-        }
-        return strBuilder.toString();
-    }
-}
diff --git a/zap/src/test/java/org/zaproxy/zap/spider/parser/SpiderSitemapXMLParserUnitTest.java b/zap/src/test/java/org/zaproxy/zap/spider/parser/SpiderSitemapXMLParserUnitTest.java
deleted file mode 100644
index de7251c5d7e..00000000000
--- a/zap/src/test/java/org/zaproxy/zap/spider/parser/SpiderSitemapXMLParserUnitTest.java
+++ /dev/null
@@ -1,320 +0,0 @@
-/*
- * Zed Attack Proxy (ZAP) and its related class files.
- *
- * ZAP is an HTTP/HTTPS proxy for assessing web application security.
- *
- * Copyright 2016 The ZAP Development Team
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.zaproxy.zap.spider.parser;
-
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.Matchers.contains;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.is;
-import static org.junit.jupiter.api.Assertions.assertThrows;
-
-import java.nio.file.Path;
-import net.htmlparser.jericho.Source;
-import org.junit.jupiter.api.Test;
-import org.parosproxy.paros.network.HttpMessage;
-
-/** Unit test for {@link SpiderSitemapXMLParser}. */
-@SuppressWarnings("deprecation")
-class SpiderSitemapXMLParserUnitTest extends SpiderParserTestUtils {
-
-    private static final String ROOT_PATH = "/";
-    private static final int BASE_DEPTH = 0;
-
-    private static final Path BASE_DIR_TEST_FILES =
-            getResourcePath("sitemapxml", SpiderSitemapXMLParserUnitTest.class);
-
-    @Test
-    void shouldFailToCreateParserWithUndefinedSpiderOptions() {
-        // Given
-        org.zaproxy.zap.spider.SpiderParam undefinedSpiderOptions = null;
-        // When / Then
-        assertThrows(
-                NullPointerException.class,
-                () ->
-                        new org.zaproxy.zap.spider.parser.SpiderSitemapXMLParser(
-                                undefinedSpiderOptions));
-    }
-
-    @Test
-    void shouldNotFailToEvaluateAnUndefinedMessage() {
-        // Given
-        HttpMessage undefinedMessage = null;
-        org.zaproxy.zap.spider.parser.SpiderSitemapXMLParser spiderParser =
-                createSpiderSitemapXMLParser();
-        // When
-        boolean canParse = spiderParser.canParseResource(undefinedMessage, ROOT_PATH, false);
-        // Then
-        assertThat(canParse, is(equalTo(false)));
-    }
-
-    @Test
-    void shouldFailToEvaluateAnUndefinedPath() {
-        // Given
-        String undefinedPath = null;
-        org.zaproxy.zap.spider.parser.SpiderSitemapXMLParser spiderParser =
-                createSpiderSitemapXMLParser();
-        // When / Then
-        assertThrows(
-                NullPointerException.class,
-                () -> spiderParser.canParseResource(new HttpMessage(), undefinedPath, false));
-    }
-
-    @Test
-    void shouldParsePathThatEndsWithSitemapXml() {
-        // Given
-        org.zaproxy.zap.spider.parser.SpiderSitemapXMLParser spiderParser =
-                createSpiderSitemapXMLParser();
-        boolean parsed = false;
-        // When
-        boolean canParse = spiderParser.canParseResource(new HttpMessage(), "/sitemap.xml", parsed);
-        // Then
-        assertThat(canParse, is(equalTo(true)));
-    }
-
-    @Test
-    void shouldParseMessageEvenIfAlreadyParsed() {
-        // Given
-        org.zaproxy.zap.spider.parser.SpiderSitemapXMLParser spiderParser =
-                createSpiderSitemapXMLParser();
-        boolean parsed = true;
-        // When
-        boolean canParse = spiderParser.canParseResource(new HttpMessage(), "/sitemap.xml", parsed);
-        // Then
-        assertThat(canParse, is(equalTo(true)));
-    }
-
-    @Test
-    void shouldNotParseAnUndefinedMessage() {
-        // Given
-        HttpMessage undefinedMessage = null;
-        org.zaproxy.zap.spider.parser.SpiderSitemapXMLParser spiderParser =
-                createSpiderSitemapXMLParser();
-        // When
-        boolean completelyParsed =
-                spiderParser.parseResource(undefinedMessage, new Source(""), BASE_DEPTH);
-        // Then
-        assertThat(completelyParsed, is(equalTo(false)));
-    }
-
-    @Test
-    void shouldNotRequireSourceToParseMessage() {
-        // Given
-        Source undefinedSource = null;
-        org.zaproxy.zap.spider.parser.SpiderSitemapXMLParser spiderParser =
-                createSpiderSitemapXMLParser();
-        HttpMessage message = createMessageWith("NoUrlsSitemap.xml");
-        // When
-        boolean completelyParsed = spiderParser.parseResource(message, undefinedSource, BASE_DEPTH);
-        // Then
-        assertThat(completelyParsed, is(equalTo(true)));
-    }
-
-    @Test
-    void shouldNotParseMessageIfParseOfSitemapXmlIsDisabled() {
-        // Given
-        org.zaproxy.zap.spider.SpiderParam params = createSpiderParamWithConfig();
-        params.setParseSitemapXml(false);
-        org.zaproxy.zap.spider.parser.SpiderSitemapXMLParser spiderParser =
-                new org.zaproxy.zap.spider.parser.SpiderSitemapXMLParser(params);
-        HttpMessage message = createMessageWith("NoUrlsSitemap.xml");
-        // When
-        boolean completelyParsed = spiderParser.parseResource(message, null, BASE_DEPTH);
-        // Then
-        assertThat(completelyParsed, is(equalTo(false)));
-    }
-
-    @Test
-    void shouldNotParseNonXmlMessage() {
-        // Given
-        org.zaproxy.zap.spider.parser.SpiderSitemapXMLParser spiderParser =
-                createSpiderSitemapXMLParser();
-        HttpMessage message = createMessageWith("text/html", "NoUrlsSitemap.xml");
-        // When
-        boolean completelyParsed = spiderParser.parseResource(message, null, BASE_DEPTH);
-        // Then
-        assertThat(completelyParsed, is(equalTo(false)));
-    }
-
-    @Test
-    void shouldNotParseXmlMessageIfClientError() {
-        // Given
-        org.zaproxy.zap.spider.parser.SpiderSitemapXMLParser spiderParser =
-                createSpiderSitemapXMLParser();
-        HttpMessage message = createMessageWith("404 Not Found", "text/xml", "NoUrlsSitemap.xml");
-        // When
-        boolean completelyParsed = spiderParser.parseResource(message, null, BASE_DEPTH);
-        // Then
-        assertThat(completelyParsed, is(equalTo(false)));
-    }
-
-    @Test
-    void shouldNotParseXmlMessageIfServerError() {
-        // Given
-        org.zaproxy.zap.spider.parser.SpiderSitemapXMLParser spiderParser =
-                createSpiderSitemapXMLParser();
-        HttpMessage message =
-                createMessageWith("500 Internal Server Error", "text/xml", "NoUrlsSitemap.xml");
-        // When
-        boolean completelyParsed = spiderParser.parseResource(message, null, BASE_DEPTH);
-        // Then
-        assertThat(completelyParsed, is(equalTo(false)));
-    }
-
-    @Test
-    void shouldNotParseEmptyXmlMessage() {
-        // Given
-        org.zaproxy.zap.spider.parser.SpiderSitemapXMLParser spiderParser =
-                createSpiderSitemapXMLParser();
-        HttpMessage message = createMessageWith("EmptyFile.xml");
-        // When
-        boolean completelyParsed = spiderParser.parseResource(message, null, BASE_DEPTH);
-        // Then
-        assertThat(completelyParsed, is(equalTo(false)));
-    }
-
-    @Test
-    void shouldNotParseMalformedXmlMessage() {
-        // Given
-        org.zaproxy.zap.spider.parser.SpiderSitemapXMLParser spiderParser =
-                createSpiderSitemapXMLParser();
-        HttpMessage message = createMessageWith("MalformedSitemap.xml");
-        // When
-        boolean completelyParsed = spiderParser.parseResource(message, null, BASE_DEPTH);
-        // Then
-        assertThat(completelyParsed, is(equalTo(false)));
-    }
-
-    @Test
-    void shouldNotParseXmlMessageWithDoctype() {
-        // Given
-        org.zaproxy.zap.spider.parser.SpiderSitemapXMLParser spiderParser =
-                createSpiderSitemapXMLParser();
-        HttpMessage message = createMessageWith("DoctypeSitemap.xml");
-        // When
-        boolean completelyParsed = spiderParser.parseResource(message, null, BASE_DEPTH);
-        // Then
-        assertThat(completelyParsed, is(equalTo(false)));
-    }
-
-    @Test
-    void shouldNotFindUrlsIfNoneDefinedInSitemap() {
-        // Given
-        org.zaproxy.zap.spider.parser.SpiderSitemapXMLParser spiderParser =
-                createSpiderSitemapXMLParser();
-        TestSpiderParserListener listener = createTestSpiderParserListener();
-        spiderParser.addSpiderParserListener(listener);
-        HttpMessage message = createMessageWith("NoUrlsSitemap.xml");
-        // When
-        boolean completelyParsed = spiderParser.parseResource(message, null, BASE_DEPTH);
-        // Then
-        assertThat(completelyParsed, is(equalTo(true)));
-        assertThat(listener.getNumberOfUrlsFound(), is(equalTo(0)));
-    }
-
-    @Test
-    void shouldNotFindUrlsIfUrlHasNoLocationIsEmptyInSitemap() {
-        // Given
-        org.zaproxy.zap.spider.parser.SpiderSitemapXMLParser spiderParser =
-                createSpiderSitemapXMLParser();
-        TestSpiderParserListener listener = createTestSpiderParserListener();
-        spiderParser.addSpiderParserListener(listener);
-        HttpMessage message = createMessageWith("UrlNoLocationSitemap.xml");
-        // When
-        boolean completelyParsed = spiderParser.parseResource(message, null, BASE_DEPTH);
-        // Then
-        assertThat(completelyParsed, is(equalTo(true)));
-        assertThat(listener.getNumberOfUrlsFound(), is(equalTo(0)));
-    }
-
-    @Test
-    void shouldNotFindUrlsIfUrlLocationIsEmptyInSitemap() {
-        // Given
-        org.zaproxy.zap.spider.parser.SpiderSitemapXMLParser spiderParser =
-                createSpiderSitemapXMLParser();
-        TestSpiderParserListener listener = createTestSpiderParserListener();
-        spiderParser.addSpiderParserListener(listener);
-        HttpMessage message = createMessageWith("UrlEmptyLocationSitemap.xml");
-        // When
-        boolean completelyParsed = spiderParser.parseResource(message, null, BASE_DEPTH);
-        // Then
-        assertThat(completelyParsed, is(equalTo(true)));
-        assertThat(listener.getNumberOfUrlsFound(), is(equalTo(0)));
-    }
-
-    @Test
-    void shouldFindUrlsInValidSitemapXml() throws Exception {
-        // Given
-        org.zaproxy.zap.spider.parser.SpiderSitemapXMLParser spiderParser =
-                createSpiderSitemapXMLParser();
-        TestSpiderParserListener listener = createTestSpiderParserListener();
-        spiderParser.addSpiderParserListener(listener);
-        HttpMessage message = createMessageWith("MultipleUrlsSitemap.xml");
-        // When
-        boolean completelyParsed = spiderParser.parseResource(message, null, BASE_DEPTH);
-        // Then
-        assertThat(completelyParsed, is(equalTo(true)));
-        assertThat(listener.getNumberOfUrlsFound(), is(equalTo(5)));
-        assertThat(
-                listener.getUrlsFound(),
-                contains(
-                        "https://example.org/",
-                        "http://subdomain.example.com/",
-                        "http://example.com/relative",
-                        "ftp://example.com/",
-                        "http://www.example.com/%C7"));
-    }
-
-    private static org.zaproxy.zap.spider.parser.SpiderSitemapXMLParser
-            createSpiderSitemapXMLParser() {
-        org.zaproxy.zap.spider.SpiderParam params = createSpiderParamWithConfig();
-        params.setParseSitemapXml(true);
-        return new org.zaproxy.zap.spider.parser.SpiderSitemapXMLParser(params);
-    }
-
-    private static HttpMessage createMessageWith(String filename) {
-        return createMessageWith("text/xml", filename);
-    }
-
-    private static HttpMessage createMessageWith(String contentType, String filename) {
-        return createMessageWith("200 OK", contentType, filename);
-    }
-
-    private static HttpMessage createMessageWith(
-            String statusCodeMessage, String contentType, String filename) {
-        HttpMessage message = new HttpMessage();
-        try {
-            String fileContents = readFile(BASE_DIR_TEST_FILES.resolve(filename));
-            message.setRequestHeader("GET / HTTP/1.1\r\nHost: example.com\r\n");
-            message.setResponseHeader(
-                    "HTTP/1.1 "
-                            + statusCodeMessage
-                            + "\r\n"
-                            + "Content-Type: "
-                            + contentType
-                            + "; charset=UTF-8\r\n"
-                            + "Content-Length: "
-                            + fileContents.length());
-            message.setResponseBody(fileContents);
-        } catch (Exception e) {
-            throw new RuntimeException(e);
-        }
-        return message;
-    }
-}
diff --git a/zap/src/test/java/org/zaproxy/zap/spider/parser/SpiderTextParserUnitTest.java b/zap/src/test/java/org/zaproxy/zap/spider/parser/SpiderTextParserUnitTest.java
deleted file mode 100644
index 261962d34f7..00000000000
--- a/zap/src/test/java/org/zaproxy/zap/spider/parser/SpiderTextParserUnitTest.java
+++ /dev/null
@@ -1,251 +0,0 @@
-/*
- * Zed Attack Proxy (ZAP) and its related class files.
- *
- * ZAP is an HTTP/HTTPS proxy for assessing web application security.
- *
- * Copyright 2016 The ZAP Development Team
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.zaproxy.zap.spider.parser;
-
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.Matchers.contains;
-import static org.hamcrest.Matchers.empty;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.is;
-import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.mockito.Mockito.mock;
-
-import net.htmlparser.jericho.Source;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.parosproxy.paros.network.HttpMessage;
-
-/** Unit test for {@link SpiderTextParser}. */
-@SuppressWarnings("deprecation")
-class SpiderTextParserUnitTest extends SpiderParserTestUtils {
-
-    private static final String EMPTY_BODY = "";
-    private static final String ROOT_PATH = "/";
-    private static final int BASE_DEPTH = 0;
-
-    private SpiderTextParser spiderParser;
-
-    @BeforeEach
-    void setup() {
-        org.zaproxy.zap.spider.SpiderParam undefinedSpiderOptions =
-                mock(org.zaproxy.zap.spider.SpiderParam.class);
-        spiderParser = new org.zaproxy.zap.spider.parser.SpiderTextParser(undefinedSpiderOptions);
-    }
-
-    @Test
-    void shouldFailToEvaluateAnUndefinedMessage() {
-        // Given
-        HttpMessage undefinedMessage = null;
-        // When / Then
-        assertThrows(
-                NullPointerException.class,
-                () -> spiderParser.canParseResource(undefinedMessage, ROOT_PATH, false));
-    }
-
-    @Test
-    void shouldNotParseMessageIfAlreadyParsed() {
-        // Given
-        boolean parsed = true;
-        // When
-        boolean canParse = spiderParser.canParseResource(new HttpMessage(), ROOT_PATH, parsed);
-        // Then
-        assertThat(canParse, is(equalTo(false)));
-    }
-
-    @Test
-    void shouldNotParseNonTextResponse() {
-        // Given
-        HttpMessage message = createMessageWith("application/xyz", EMPTY_BODY);
-        boolean parsed = false;
-        // When
-        boolean canParse = spiderParser.canParseResource(message, ROOT_PATH, parsed);
-        // Then
-        assertThat(canParse, is(equalTo(false)));
-    }
-
-    @Test
-    void shouldNotParseTextHtmlResponse() {
-        // Given
-        HttpMessage message = createMessageWith("text/html", EMPTY_BODY);
-        boolean parsed = false;
-        // When
-        boolean canParse = spiderParser.canParseResource(message, ROOT_PATH, parsed);
-        // Then
-        assertThat(canParse, is(equalTo(false)));
-    }
-
-    @Test
-    void shouldParseTextResponse() {
-        // Given
-        HttpMessage messageHtmlResponse = createMessageWith(EMPTY_BODY);
-        boolean parsed = false;
-        // When
-        boolean canParse = spiderParser.canParseResource(messageHtmlResponse, ROOT_PATH, parsed);
-        // Then
-        assertThat(canParse, is(equalTo(true)));
-    }
-
-    @Test
-    void shouldParseTextResponseEvenIfProvidedPathIsNull() {
-        // Given
-        HttpMessage messageHtmlResponse = createMessageWith(EMPTY_BODY);
-        boolean parsed = false;
-        // When
-        boolean canParse = spiderParser.canParseResource(messageHtmlResponse, null, parsed);
-        // Then
-        assertThat(canParse, is(equalTo(true)));
-    }
-
-    @Test
-    void shouldNotParseTextResponseIfAlreadyParsed() {
-        // Given
-        HttpMessage messageHtmlResponse = createMessageWith(EMPTY_BODY);
-        boolean parsed = true;
-        // When
-        boolean canParse = spiderParser.canParseResource(messageHtmlResponse, ROOT_PATH, parsed);
-        // Then
-        assertThat(canParse, is(equalTo(false)));
-    }
-
-    @Test
-    void shouldFailToParseAnUndefinedMessage() {
-        // Given
-        HttpMessage undefinedMessage = null;
-        Source source = createSource(createMessageWith(EMPTY_BODY));
-        // When / Then
-        assertThrows(
-                NullPointerException.class,
-                () -> spiderParser.parseResource(undefinedMessage, source, BASE_DEPTH));
-    }
-
-    @Test
-    void shouldNeverConsiderCompletelyParsed() {
-        // Given
-        HttpMessage message = createMessageWith("Non Empty Body...");
-        Source source = createSource(message);
-        // When
-        boolean completelyParsed = spiderParser.parseResource(message, source, BASE_DEPTH);
-        // Then
-        assertThat(completelyParsed, is(equalTo(false)));
-    }
-
-    @Test
-    void shouldNotFindUrlsIfThereIsNone() {
-        // Given
-        TestSpiderParserListener listener = createTestSpiderParserListener();
-        spiderParser.addSpiderParserListener(listener);
-        HttpMessage message =
-                createMessageWith(
-                        body(
-                                "Body with no HTTP/S URLs",
-                                " ://example.com/ ",
-                                "More text...  ftp://ftp.example.com/ ",
-                                "Even more text... //noscheme.example.com "));
-        Source source = createSource(message);
-        // When
-        boolean completelyParsed = spiderParser.parseResource(message, source, BASE_DEPTH);
-        // Then
-        assertThat(completelyParsed, is(equalTo(false)));
-        assertThat(listener.getNumberOfUrlsFound(), is(equalTo(0)));
-        assertThat(listener.getUrlsFound(), is(empty()));
-    }
-
-    @Test
-    void shouldFindUrlsInCommentsWithoutElements() {
-        // Given
-        TestSpiderParserListener listener = createTestSpiderParserListener();
-        spiderParser.addSpiderParserListener(listener);
-        HttpMessage messageHtmlResponse =
-                createMessageWith(
-                        body(
-                                "Body with HTTP/S URLs",
-                                " - http://plaincomment.example.com some text not part of URL",
-                                "- \"https://plaincomment.example.com/z.php?x=y\" more text not part of URL",
-                                "- 'http://plaincomment.example.com/c.pl?x=y' even more text not part of URL",
-                                "-  ...",
-                                "- http://plaincomment.example.com/e/e1/e2.html?x=y#stop fragment should be ignored",
-                                "- (https://plaincomment.example.com/surrounded/with/parenthesis) parenthesis should not be included",
-                                "- [https://plaincomment.example.com/surrounded/with/brackets] brackets should not be included",
-                                "- {https://plaincomment.example.com/surrounded/with/curly/brackets} curly brackets should not be included",
-                                "- mixed case URLs HtTpS://ExAmPlE.CoM/path/ should also be found"));
-        Source source = createSource(messageHtmlResponse);
-        // When
-        boolean completelyParsed =
-                spiderParser.parseResource(messageHtmlResponse, source, BASE_DEPTH);
-        // Then
-        assertThat(completelyParsed, is(equalTo(false)));
-        assertThat(listener.getNumberOfUrlsFound(), is(equalTo(9)));
-        assertThat(
-                listener.getUrlsFound(),
-                contains(
-                        "http://plaincomment.example.com/",
-                        "https://plaincomment.example.com/z.php?x=y",
-                        "http://plaincomment.example.com/c.pl?x=y",
-                        "https://plaincomment.example.com/d.asp?x=y",
-                        "http://plaincomment.example.com/e/e1/e2.html?x=y",
-                        "https://plaincomment.example.com/surrounded/with/parenthesis",
-                        "https://plaincomment.example.com/surrounded/with/brackets",
-                        "https://plaincomment.example.com/surrounded/with/curly/brackets",
-                        "https://example.com/path/"));
-    }
-
-    private static HttpMessage createMessageWith(String body) {
-        return createMessageWith("text/xyz", body);
-    }
-
-    private static HttpMessage createMessageWith(String contentType, String body) {
-        return createMessageWith("200 OK", contentType, body);
-    }
-
-    private static HttpMessage createMessageWith(
-            String statusCodeMessage, String contentType, String body) {
-        HttpMessage message = new HttpMessage();
-        try {
-            message.setRequestHeader("GET / HTTP/1.1\r\nHost: example.com\r\n");
-            message.setResponseHeader(
-                    "HTTP/1.1 "
-                            + statusCodeMessage
-                            + "\r\n"
-                            + "Content-Type: "
-                            + contentType
-                            + "; charset=UTF-8\r\n"
-                            + "Content-Length: "
-                            + body.length());
-            message.setResponseBody(body);
-        } catch (Exception e) {
-            throw new RuntimeException(e);
-        }
-        return message;
-    }
-
-    private static String body(String... strings) {
-        if (strings == null || strings.length == 0) {
-            return "";
-        }
-        StringBuilder strBuilder = new StringBuilder(strings.length * 25);
-        for (String string : strings) {
-            if (strBuilder.length() > 0) {
-                strBuilder.append("\n");
-            }
-            strBuilder.append(string);
-        }
-        return strBuilder.toString();
-    }
-}