Skip to content

Commit 80b85f8

Browse files
yma955yma955
committed
Add proxy cache on CassandraPathDB
1 parent 779586c commit 80b85f8

File tree

9 files changed

+393
-32
lines changed

9 files changed

+393
-32
lines changed

‎common/src/main/java/org/commonjava/storage/pathmapped/model/ProxySite.java‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
/**
2+
* Copyright (C) 2019 Red Hat, Inc. (nos-devel@redhat.com)
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package org.commonjava.storage.pathmapped.model;
17+
18+
public interface ProxySite
19+
{
20+
String getSite();
21+
}

‎common/src/main/java/org/commonjava/storage/pathmapped/spi/PathDB.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,4 +104,16 @@ enum FileType {
104104
* @param fileType file, dir, or all
105105
*/
106106
void traverse( String fileSystem, String path, Consumer<PathMap> consumer, int limit, FileType fileType );
107+
108+
Set<String> getProxySitesCache();
109+
110+
boolean isProxySite( String site );
111+
112+
List<String> getProxySiteList();
113+
114+
void saveProxySite( String site );
115+
116+
void deleteProxySite( String site );
117+
118+
void deleteAllProxySite();
107119
}

‎pathdb/datastax/src/main/java/org/commonjava/storage/pathmapped/pathdb/datastax/CassandraPathDB.java‎

Lines changed: 84 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -26,13 +26,21 @@
2626
import com.datastax.driver.mapping.Mapper;
2727
import com.datastax.driver.mapping.MappingManager;
2828
import com.datastax.driver.mapping.Result;
29-
3029
import com.google.common.collect.TreeTraverser;
31-
import org.commonjava.storage.pathmapped.model.*;
32-
import org.commonjava.storage.pathmapped.pathdb.datastax.model.*;
30+
import org.commonjava.storage.pathmapped.config.PathMappedStorageConfig;
31+
import org.commonjava.storage.pathmapped.model.FileChecksum;
32+
import org.commonjava.storage.pathmapped.model.Filesystem;
33+
import org.commonjava.storage.pathmapped.model.PathMap;
34+
import org.commonjava.storage.pathmapped.model.Reclaim;
35+
import org.commonjava.storage.pathmapped.model.ReverseMap;
36+
import org.commonjava.storage.pathmapped.pathdb.datastax.model.DtxFileChecksum;
37+
import org.commonjava.storage.pathmapped.pathdb.datastax.model.DtxFilesystem;
38+
import org.commonjava.storage.pathmapped.pathdb.datastax.model.DtxPathMap;
39+
import org.commonjava.storage.pathmapped.pathdb.datastax.model.DtxProxySite;
40+
import org.commonjava.storage.pathmapped.pathdb.datastax.model.DtxReclaim;
41+
import org.commonjava.storage.pathmapped.pathdb.datastax.model.DtxReverseMap;
3342
import org.commonjava.storage.pathmapped.pathdb.datastax.util.AsyncJobExecutor;
3443
import org.commonjava.storage.pathmapped.pathdb.datastax.util.CassandraPathDBUtils;
35-
import org.commonjava.storage.pathmapped.config.PathMappedStorageConfig;
3644
import org.commonjava.storage.pathmapped.spi.PathDB;
3745
import org.commonjava.storage.pathmapped.spi.PathDBAdmin;
3846
import org.commonjava.storage.pathmapped.util.PathMapUtils;
@@ -55,11 +63,13 @@
5563
import java.util.function.Function;
5664
import java.util.stream.Collectors;
5765

58-
import static com.datastax.driver.core.ConsistencyLevel.*;
66+
import static com.datastax.driver.core.ConsistencyLevel.QUORUM;
5967
import static java.util.Collections.emptySet;
6068
import static org.apache.commons.lang.StringUtils.isNotBlank;
6169
import static org.commonjava.storage.pathmapped.pathdb.datastax.util.CassandraPathDBUtils.getHoursInDay;
62-
import static org.commonjava.storage.pathmapped.spi.PathDB.FileType.*;
70+
import static org.commonjava.storage.pathmapped.spi.PathDB.FileType.all;
71+
import static org.commonjava.storage.pathmapped.spi.PathDB.FileType.dir;
72+
import static org.commonjava.storage.pathmapped.spi.PathDB.FileType.file;
6373
import static org.commonjava.storage.pathmapped.util.PathMapUtils.ROOT_DIR;
6474

6575
public class CassandraPathDB
@@ -83,6 +93,8 @@ public class CassandraPathDB
8393

8494
private Mapper<DtxFilesystem> filesystemMapper;
8595

96+
protected Set<String> proxySitesCache = new HashSet<>();
97+
8698
private PathMappedStorageConfig config;
8799

88100
private String keyspace;
@@ -91,9 +103,12 @@ public class CassandraPathDB
91103

92104
private long reconnectDelay = 60000;
93105

94-
private PreparedStatement preparedExistQuery, preparedListQuery, preparedListCheckEmpty, preparedContainingQuery, preparedExistFileQuery,
95-
preparedUpdateExpiration, preparedReverseMapIncrement, preparedReverseMapReduction,
96-
preparedFilesystemIncrement, preparedFilesystemReduction, preparedFilesystemList;
106+
private Mapper<DtxProxySite> proxysiteMapper;
107+
108+
private PreparedStatement preparedExistQuery, preparedListQuery, preparedListCheckEmpty, preparedContainingQuery,
109+
preparedExistFileQuery, preparedUpdateExpiration, preparedReverseMapIncrement, preparedReverseMapReduction,
110+
preparedFilesystemIncrement, preparedFilesystemReduction, preparedFilesystemList, preparedProxySiteQuery,
111+
preparedProxySiteList, preparedProxySiteTruncate;
97112

98113
@Deprecated
99114
public CassandraPathDB( PathMappedStorageConfig config, Session session, String keyspace )
@@ -158,6 +173,7 @@ private void prepare( Session session, String keyspace, int replicationFactor )
158173
session.execute( CassandraPathDBUtils.getSchemaCreateTableReclaim( keyspace ) );
159174
session.execute( CassandraPathDBUtils.getSchemaCreateTableFileChecksum( keyspace ) );
160175
session.execute( CassandraPathDBUtils.getSchemaCreateTableFilesystem( keyspace ) );
176+
session.execute( CassandraPathDBUtils.getSchemaCreateTableProxySites( keyspace ) );
161177

162178
MappingManager manager = new MappingManager( session );
163179

@@ -166,6 +182,7 @@ private void prepare( Session session, String keyspace, int replicationFactor )
166182
reclaimMapper = manager.mapper( DtxReclaim.class, keyspace );
167183
fileChecksumMapper = manager.mapper( DtxFileChecksum.class, keyspace );
168184
filesystemMapper = manager.mapper( DtxFilesystem.class, keyspace );
185+
proxysiteMapper = manager.mapper( DtxProxySite.class, keyspace );
169186

170187
preparedExistFileQuery = session.prepare( "SELECT count(*) FROM " + keyspace
171188
+ ".pathmap WHERE filesystem=? and parentpath=? and filename=?;" );
@@ -185,22 +202,28 @@ private void prepare( Session session, String keyspace, int replicationFactor )
185202
"WHERE filesystem=? and parentpath=? and filename=?;" );
186203

187204
preparedContainingQuery = session.prepare( "SELECT filesystem FROM " + keyspace
188-
+ ".pathmap WHERE filesystem IN ? and parentpath=? and filename=?;" );
205+
+ ".pathmap WHERE filesystem IN ? and parentpath=? and filename=?;" );
189206

190207
preparedReverseMapIncrement =
191-
session.prepare( "UPDATE " + keyspace + ".reversemap SET paths = paths + ? WHERE fileid=?;" );
208+
session.prepare( "UPDATE " + keyspace + ".reversemap SET paths = paths + ? WHERE fileid=?;" );
192209

193210
preparedReverseMapReduction =
194-
session.prepare( "UPDATE " + keyspace + ".reversemap SET paths = paths - ? WHERE fileid=?;" );
211+
session.prepare( "UPDATE " + keyspace + ".reversemap SET paths = paths - ? WHERE fileid=?;" );
195212
preparedReverseMapReduction.setConsistencyLevel( QUORUM );
196213

197-
preparedFilesystemIncrement =
198-
session.prepare("UPDATE " + keyspace + ".filesystem SET filecount=filecount+?, size=size+? WHERE filesystem=?;" );
214+
preparedFilesystemIncrement = session.prepare(
215+
"UPDATE " + keyspace + ".filesystem SET filecount=filecount+?, size=size+? WHERE filesystem=?;" );
199216

200-
preparedFilesystemReduction =
201-
session.prepare("UPDATE " + keyspace + ".filesystem SET filecount=filecount-?, size=size-? WHERE filesystem=?;" );
217+
preparedFilesystemReduction = session.prepare(
218+
"UPDATE " + keyspace + ".filesystem SET filecount=filecount-?, size=size-? WHERE filesystem=?;" );
202219

203-
preparedFilesystemList = session.prepare("SELECT * FROM " + keyspace + ".filesystem;" );
220+
preparedFilesystemList = session.prepare( "SELECT * FROM " + keyspace + ".filesystem;" );
221+
222+
preparedProxySiteQuery = session.prepare( "SELECT * FROM " + keyspace + ".proxysites WHERE site=?;" );
223+
224+
preparedProxySiteList = session.prepare( "SELECT * FROM " + keyspace + ".proxysites;" );
225+
226+
preparedProxySiteTruncate = session.prepare( "TRUNCATE " + keyspace + ".proxysites;" );
204227

205228
asyncJobExecutor = new AsyncJobExecutor( config );
206229
}
@@ -997,4 +1020,48 @@ private ResultSet executeSession ( BoundStatement bind )
9971020
}
9981021
return trackingRecord;
9991022
}
1023+
1024+
@Override
1025+
public Set<String> getProxySitesCache()
1026+
{
1027+
return proxySitesCache;
1028+
}
1029+
1030+
@Override
1031+
public boolean isProxySite( String site )
1032+
{
1033+
BoundStatement bound = preparedProxySiteQuery.bind( site );
1034+
ResultSet result = executeSession( bound );
1035+
return notNull( result );
1036+
}
1037+
1038+
@Override
1039+
public List<String> getProxySiteList()
1040+
{
1041+
ResultSet result = executeSession( preparedProxySiteList.bind() );
1042+
return proxysiteMapper.map( result ).all().stream().map( DtxProxySite::getSite ).collect( Collectors.toList() );
1043+
}
1044+
1045+
@Override
1046+
public void saveProxySite( String site )
1047+
{
1048+
DtxProxySite proxySite = new DtxProxySite( site );
1049+
logger.debug( "ProxySite, {}", site );
1050+
proxysiteMapper.save( proxySite );
1051+
}
1052+
1053+
@Override
1054+
public void deleteProxySite( String site )
1055+
{
1056+
DtxProxySite proxySite = new DtxProxySite( site );
1057+
logger.debug( "Delete proxySite, {}", site );
1058+
proxysiteMapper.delete( proxySite );
1059+
}
1060+
1061+
@Override
1062+
public void deleteAllProxySite()
1063+
{
1064+
BoundStatement bound = preparedProxySiteTruncate.bind();
1065+
executeSession( bound );
1066+
}
10001067
}

‎pathdb/datastax/src/main/java/org/commonjava/storage/pathmapped/pathdb/datastax/model/DtxProxySite.java‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
/**
2+
* Copyright (C) 2019 Red Hat, Inc. (nos-devel@redhat.com)
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package org.commonjava.storage.pathmapped.pathdb.datastax.model;
17+
18+
import com.datastax.driver.mapping.annotations.PartitionKey;
19+
import com.datastax.driver.mapping.annotations.Table;
20+
import org.commonjava.storage.pathmapped.model.ProxySite;
21+
22+
import java.util.Objects;
23+
24+
@Table( name = "proxysites", readConsistency = "QUORUM", writeConsistency = "QUORUM" )
25+
public class DtxProxySite
26+
implements ProxySite
27+
{
28+
@PartitionKey
29+
private String site;
30+
31+
public DtxProxySite()
32+
{
33+
}
34+
35+
public DtxProxySite( String site )
36+
{
37+
this.site = site;
38+
}
39+
40+
@Override
41+
public String getSite()
42+
{
43+
return site;
44+
}
45+
46+
public void setSite( String site )
47+
{
48+
this.site = site;
49+
}
50+
51+
@Override
52+
public boolean equals( Object o )
53+
{
54+
if ( this == o )
55+
{
56+
return true;
57+
}
58+
if ( o == null || getClass() != o.getClass() )
59+
{
60+
return false;
61+
}
62+
DtxProxySite that = (DtxProxySite) o;
63+
return site.equals( that.site );
64+
}
65+
66+
@Override
67+
public int hashCode()
68+
{
69+
return Objects.hash( site );
70+
}
71+
72+
@Override
73+
public String toString()
74+
{
75+
return "DtxProxySite{" + "site='" + site + '}';
76+
}
77+
}

‎pathdb/datastax/src/main/java/org/commonjava/storage/pathmapped/pathdb/datastax/util/CassandraPathDBUtils.java‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -96,12 +96,14 @@ public static String getSchemaCreateTableReclaim( String keyspace )
9696

9797
public static String getSchemaCreateTableFileChecksum( String keyspace )
9898
{
99-
return "CREATE TABLE IF NOT EXISTS " + keyspace + ".filechecksum ("
100-
+ "checksum varchar,"
101-
+ "fileid varchar,"
102-
+ "storage varchar,"
103-
+ "PRIMARY KEY (checksum)"
104-
+ ");";
99+
return "CREATE TABLE IF NOT EXISTS " + keyspace + ".filechecksum (" + "checksum varchar," + "fileid varchar,"
100+
+ "storage varchar," + "PRIMARY KEY (checksum)" + ");";
101+
}
102+
103+
public static String getSchemaCreateTableProxySites( String keyspace )
104+
{
105+
return "CREATE TABLE IF NOT EXISTS " + keyspace + ".proxysites (" + "site varchar," + "PRIMARY KEY (site)"
106+
+ ");";
105107
}
106108

107109
// Since Date.getHours is deprecated, we use this to replace it.

‎pathdb/jpa/src/main/java/org/commonjava/storage/pathmapped/pathdb/jpa/JPAPathDB.java‎

Lines changed: 39 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -15,24 +15,24 @@
1515
*/
1616
package org.commonjava.storage.pathmapped.pathdb.jpa;
1717

18+
import jakarta.persistence.EntityManager;
19+
import jakarta.persistence.EntityManagerFactory;
20+
import jakarta.persistence.Persistence;
21+
import jakarta.persistence.Query;
1822
import org.commonjava.storage.pathmapped.model.FileChecksum;
23+
import org.commonjava.storage.pathmapped.model.PathMap;
24+
import org.commonjava.storage.pathmapped.model.Reclaim;
25+
import org.commonjava.storage.pathmapped.model.ReverseMap;
1926
import org.commonjava.storage.pathmapped.pathdb.jpa.model.JpaPathKey;
2027
import org.commonjava.storage.pathmapped.pathdb.jpa.model.JpaPathMap;
2128
import org.commonjava.storage.pathmapped.pathdb.jpa.model.JpaReclaim;
2229
import org.commonjava.storage.pathmapped.pathdb.jpa.model.JpaReverseKey;
2330
import org.commonjava.storage.pathmapped.pathdb.jpa.model.JpaReverseMap;
24-
import org.commonjava.storage.pathmapped.model.PathMap;
25-
import org.commonjava.storage.pathmapped.model.Reclaim;
26-
import org.commonjava.storage.pathmapped.model.ReverseMap;
2731
import org.commonjava.storage.pathmapped.spi.PathDB;
2832
import org.commonjava.storage.pathmapped.util.PathMapUtils;
2933
import org.slf4j.Logger;
3034
import org.slf4j.LoggerFactory;
3135

32-
import jakarta.persistence.EntityManager;
33-
import jakarta.persistence.EntityManagerFactory;
34-
import jakarta.persistence.Persistence;
35-
import jakarta.persistence.Query;
3636
import java.util.ArrayList;
3737
import java.util.Collection;
3838
import java.util.Collections;
@@ -437,4 +437,36 @@ private JpaPathKey getPathKey( String fileSystem, String path )
437437
return new JpaPathKey( fileSystem, parentPath, filename );
438438
}
439439

440+
@Override
441+
public Set<String> getProxySitesCache()
442+
{
443+
return null;
444+
}
445+
446+
@Override
447+
public boolean isProxySite( String site )
448+
{
449+
return false;
450+
}
451+
452+
@Override
453+
public List<String> getProxySiteList()
454+
{
455+
return null;
456+
}
457+
458+
@Override
459+
public void saveProxySite( String site )
460+
{
461+
}
462+
463+
@Override
464+
public void deleteProxySite( String site )
465+
{
466+
}
467+
468+
@Override
469+
public void deleteAllProxySite()
470+
{
471+
}
440472
}

0 commit comments

Comments
 (0)