Skip to content

Commit 44af866

Browse files
frankdevhubfrankdevhub
committed
parse sitemap and push page links to seo platform
1 parent 8a37b53 commit 44af866

9 files changed

Lines changed: 630 additions & 478 deletions

File tree

‎plugin/site-plugin/pom.xml‎

Lines changed: 372 additions & 368 deletions
Large diffs are not rendered by default.

‎plugin/site-plugin/src/main/java/com/frankdevhub/site/core/utils/RequestBodyUtils.java‎

Lines changed: 67 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -11,78 +11,78 @@
1111

1212
public class RequestBodyUtils {
1313

14-
private static final boolean isJson(String json) {
15-
try {
16-
JSONObject.parseObject(json);
17-
return true;
18-
} catch (Exception e) {
19-
return false;
20-
}
21-
}
14+
private static final boolean isJson(String json) {
15+
try {
16+
JSONObject.parseObject(json);
17+
return true;
18+
} catch (Exception e) {
19+
return false;
20+
}
21+
}
2222

23-
private static boolean isJsonArray(String json) throws JSONException {
24-
Object object = new JSONTokener(json).nextValue();
25-
if (object instanceof org.json.JSONArray)
26-
return true;
27-
else
28-
return false;
29-
}
23+
private static boolean isJsonArray(String json) throws JSONException {
24+
Object object = new JSONTokener(json).nextValue();
25+
if (object instanceof org.json.JSONArray)
26+
return true;
27+
else
28+
return false;
29+
}
3030

31-
private static String decryptBody(String json) {
32-
JSONObject obj = JSON.parseObject(json);
33-
decryptBody(obj);
34-
return obj.toJSONString();
35-
}
31+
private static String decryptBody(String json) throws JSONException {
32+
JSONObject obj = JSON.parseObject(json);
33+
decryptBody(obj);
34+
return obj.toJSONString();
35+
}
3636

37-
public static String parseBody(String json, boolean decrypt) {
38-
JSONObject obj = JSON.parseObject(json);
39-
parseBody(obj, decrypt);
40-
return obj.toJSONString();
41-
}
37+
public static String parseBody(String json, boolean decrypt) throws JSONException {
38+
JSONObject obj = JSON.parseObject(json);
39+
parseBody(obj, decrypt);
40+
return obj.toJSONString();
41+
}
4242

43-
private static void parseBody(JSONArray array, boolean decrypt) {
44-
Iterator iter = array.iterator();
45-
while (iter.hasNext()) {
46-
JSONObject jsonObject = (JSONObject) iter.next();
47-
parseBody(jsonObject, decrypt);
48-
}
49-
}
43+
private static void parseBody(JSONArray array, boolean decrypt) throws JSONException {
44+
Iterator iter = array.iterator();
45+
while (iter.hasNext()) {
46+
JSONObject jsonObject = (JSONObject) iter.next();
47+
parseBody(jsonObject, decrypt);
48+
}
49+
}
5050

51-
private static void parseBody(JSONObject parent, boolean decrypt) {
52-
Iterator iter = parent.entrySet().iterator();
53-
while (iter.hasNext()) {
54-
Map.Entry entry = (Map.Entry) iter.next();
55-
String e = entry.getKey().toString();
56-
String v = entry.getValue().toString();
57-
if (!isJsonArray(v)) {
58-
if (decrypt)
59-
parent.put(e, AESEncrypt.decryptBase64(v));
60-
else
61-
parent.put(e, AESEncrypt.encryptBase64(v));
62-
} else {
63-
System.out.println(v);
64-
JSONArray array = JSONArray.parseArray(v);
65-
parseBody(array, decrypt);
66-
parent.put(e, array);
67-
}
68-
}
69-
}
51+
private static void parseBody(JSONObject parent, boolean decrypt) throws JSONException {
52+
Iterator iter = parent.entrySet().iterator();
53+
while (iter.hasNext()) {
54+
Map.Entry entry = (Map.Entry) iter.next();
55+
String e = entry.getKey().toString();
56+
String v = entry.getValue().toString();
57+
if (!isJsonArray(v)) {
58+
if (decrypt)
59+
parent.put(e, AESEncrypt.decryptBase64(v));
60+
else
61+
parent.put(e, AESEncrypt.encryptBase64(v));
62+
} else {
63+
System.out.println(v);
64+
JSONArray array = JSONArray.parseArray(v);
65+
parseBody(array, decrypt);
66+
parent.put(e, array);
67+
}
68+
}
69+
}
7070

71-
private static void decryptBody(JSONObject parent) {
72-
Iterator iter = parent.entrySet().iterator();
73-
while (iter.hasNext()) {
74-
Map.Entry entry = (Map.Entry) iter.next();
71+
private static void decryptBody(JSONObject parent) throws JSONException {
72+
Iterator iter = parent.entrySet().iterator();
73+
while (iter.hasNext()) {
74+
Map.Entry entry = (Map.Entry) iter.next();
7575

76-
String e = entry.getKey().toString();
77-
String v = entry.getValue().toString();
78-
if (!isJsonArray(v))
79-
parent.put(e, AESEncrypt.decryptBase64(v));
80-
else {
81-
v = v.substring(1, v.lastIndexOf("]"));
82-
JSONObject sub = JSON.parseObject(v);
83-
parent.put(e, sub);
84-
decryptBody(sub);
85-
}
86-
}
87-
}
76+
String e = entry.getKey().toString();
77+
String v = entry.getValue().toString();
78+
if (!isJsonArray(v))
79+
parent.put(e, AESEncrypt.decryptBase64(v));
80+
else {
81+
v = v.substring(1, v.lastIndexOf("]"));
82+
JSONObject sub = JSON.parseObject(v);
83+
parent.put(e, sub);
84+
decryptBody(sub);
85+
}
86+
}
87+
}
8888
}

‎plugin/site-plugin/src/main/java/com/frankdevhub/site/core/utils/SiteMapParseUtils.java‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
package com.frankdevhub.site.core.utils;
2+
3+
import java.io.File;
4+
import java.util.ArrayList;
5+
import java.util.HashMap;
6+
import java.util.List;
7+
import java.util.Map;
8+
9+
import org.dom4j.Document;
10+
import org.dom4j.DocumentException;
11+
import org.dom4j.Element;
12+
import org.dom4j.Node;
13+
import org.dom4j.io.SAXReader;
14+
import org.springframework.util.Assert;
15+
16+
public class SiteMapParseUtils {
17+
18+
public Map<Object, Object> parseXMLDocument(File file) throws DocumentException {
19+
Map<Object, Object> res = new HashMap<>();
20+
long start = System.currentTimeMillis();
21+
Document doc = new SAXReader().read(file);
22+
String xpath = "//urlset";
23+
Element root = (Element) doc.selectSingleNode(xpath);
24+
Assert.notNull(root, "cannot search root node for curent sitemap. xapth=//urlset");
25+
26+
String namespace = root.getNamespaceURI();
27+
xpath = "//*[local-name()='url' and namespace-uri()='" + namespace + "']" + "/*[local-name()='loc']";
28+
List<Node> nodes = doc.selectNodes(xpath);
29+
int size = nodes.size();
30+
List<String> urls = new ArrayList<>();
31+
32+
for (Node n : nodes) {
33+
System.out.println("page url: " + n.getText());
34+
urls.add(n.getText());
35+
}
36+
long end = System.currentTimeMillis();
37+
double cost = (end - start) / 1000;
38+
res.put("timecost", cost);
39+
res.put("size", size);
40+
res.put("list", urls);
41+
return res;
42+
}
43+
44+
public Map<Object, Object> pushToBaiduPlatform(List<String> urls) {
45+
Map<Object, Object> res = new HashMap<>();
46+
47+
return res;
48+
}
49+
50+
private boolean pushToBaiduPlatform(String url) {
51+
boolean succeed = false;
52+
return succeed;
53+
}
54+
55+
public static void main(String[] args) throws Exception {
56+
File file = new File("D://sitemap.xml");
57+
if (!file.exists())
58+
throw new Exception("file not exists");
59+
new SiteMapParseUtils().parseXMLDocument(file);
60+
61+
}
62+
}

‎plugin/site-plugin/src/main/java/com/frankdevhub/site/entities/BaseRecord.java‎

Lines changed: 34 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -5,38 +5,38 @@
55

66
public class BaseRecord<T> {
77

8-
@Column(name = "create_time")
9-
private Long createTime;
10-
11-
@Column(name = "update_time")
12-
private Long updateTime;
13-
14-
public Long getCreateTime() {
15-
return createTime;
16-
}
17-
18-
private BaseRecord setCreateTime(Long createTime) {
19-
this.createTime = createTime;
20-
return this;
21-
}
22-
23-
public Long getUpdateTime() {
24-
return updateTime;
25-
}
26-
27-
private BaseRecord setUpdateTime(Long updateTime) {
28-
this.updateTime = updateTime;
29-
return this;
30-
}
31-
32-
public T doCreateEntity() {
33-
Long timeStamp = new Date().getTime();
34-
this.setCreateTime(timeStamp).setUpdateTime(timeStamp);
35-
return (T) this;
36-
}
37-
38-
public T doUpdateEntity() {
39-
this.setUpdateTime(new Date().getTime());
40-
return (T) this;
41-
}
8+
@Column(name = "create_time")
9+
private Long createTime;
10+
11+
@Column(name = "update_time")
12+
private Long updateTime;
13+
14+
public Long getCreateTime() {
15+
return createTime;
16+
}
17+
18+
private BaseRecord setCreateTime(Long createTime) {
19+
this.createTime = createTime;
20+
return this;
21+
}
22+
23+
public Long getUpdateTime() {
24+
return updateTime;
25+
}
26+
27+
private BaseRecord setUpdateTime(Long updateTime) {
28+
this.updateTime = updateTime;
29+
return this;
30+
}
31+
32+
public T doCreateEntity() {
33+
Long timeStamp = new Date().getTime();
34+
this.setCreateTime(timeStamp).setUpdateTime(timeStamp);
35+
return (T) this;
36+
}
37+
38+
public T doUpdateEntity() {
39+
this.setUpdateTime(new Date().getTime());
40+
return (T) this;
41+
}
4242
}

‎plugin/site-plugin/src/main/java/com/frankdevhub/site/entities/SiteMapPushRecord.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
package com.frankdevhub.site.entities;
2+
3+
import javax.persistence.Column;
4+
import javax.persistence.Id;
5+
import javax.persistence.Table;
6+
7+
@Table(name = "site_map_push_record")
8+
public class SiteMapPushRecord extends BaseRecord<SiteMapPushRecord> {
9+
10+
@Id
11+
@Column(name = "id")
12+
private Long id;
13+
14+
@Column(name = "record_id")
15+
private Long recordId;
16+
17+
}

‎plugin/site-plugin/src/main/java/com/frankdevhub/site/mapper/SiteMapPushRecordMapper.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
package com.frankdevhub.site.mapper;
2+
3+
import org.apache.ibatis.annotations.Mapper;
4+
5+
import com.frankdevhub.site.core.utils.ViewMapper;
6+
import com.frankdevhub.site.entities.SiteMapPushRecord;
7+
8+
@Mapper
9+
public interface SiteMapPushRecordMapper extends ViewMapper<SiteMapPushRecord> {
10+
11+
}

‎plugin/site-plugin/src/main/java/com/frankdevhub/site/repository/PageLoggerIpRepository.java‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,20 @@
11
package com.frankdevhub.site.repository;
22

3-
43
import com.frankdevhub.site.core.repository.MyBatisRepository;
54
import com.frankdevhub.site.core.utils.SpringUtils;
65
import com.frankdevhub.site.entities.PageLoggerIpEntity;
76
import com.frankdevhub.site.mapper.PageLoggerIpMapper;
87
import org.springframework.stereotype.Repository;
98

10-
119
@Repository
1210
public class PageLoggerIpRepository extends MyBatisRepository {
1311

14-
private PageLoggerIpMapper getMapper() {
15-
return SpringUtils.getBean(PageLoggerIpMapper.class);
16-
}
12+
private PageLoggerIpMapper getMapper() {
13+
return SpringUtils.getBean(PageLoggerIpMapper.class);
14+
}
1715

18-
public Integer insertSelective(PageLoggerIpEntity entity) {
19-
entity.doCreateEntity();
20-
return getMapper().insertSelective(entity);
21-
}
16+
public Integer insertSelective(PageLoggerIpEntity entity) {
17+
entity.doCreateEntity();
18+
return getMapper().insertSelective(entity);
19+
}
2220
}

‎plugin/site-plugin/src/main/java/com/frankdevhub/site/repository/SiteMapPushRecordRepository.java‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
package com.frankdevhub.site.repository;
2+
3+
import org.springframework.stereotype.Repository;
4+
5+
import com.frankdevhub.site.core.repository.MyBatisRepository;
6+
import com.frankdevhub.site.core.utils.SpringUtils;
7+
import com.frankdevhub.site.entities.SiteMapPushRecord;
8+
import com.frankdevhub.site.mapper.SiteMapPushRecordMapper;
9+
10+
@Repository
11+
public class SiteMapPushRecordRepository extends MyBatisRepository {
12+
13+
private SiteMapPushRecordMapper getMapper() {
14+
return SpringUtils.getBean(SiteMapPushRecordMapper.class);
15+
}
16+
17+
public Integer insertSelective(SiteMapPushRecord entity) {
18+
entity.doCreateEntity();
19+
return getMapper().insertSelective(entity);
20+
}
21+
}

0 commit comments

Comments
 (0)