diff --git a/core/src/main/java/com/gitee/kooder/code/CodeFileTraveler.java b/core/src/main/java/com/gitee/kooder/code/CodeFileTraveler.java index 8c781d6112c83168db1cf3b9be61f93388c0004d..ebae2d677cf9f339e28f2cdd3a6e3570820afbcf 100644 --- a/core/src/main/java/com/gitee/kooder/code/CodeFileTraveler.java +++ b/core/src/main/java/com/gitee/kooder/code/CodeFileTraveler.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.code; import com.gitee.kooder.core.Constants; diff --git a/core/src/main/java/com/gitee/kooder/code/FileRepositoryProvider.java b/core/src/main/java/com/gitee/kooder/code/FileRepositoryProvider.java index c22eb971d3b3f0fb001cd4c1cfe8dd9c3e510675..961f2cb1e2be5b5e798b8f03cdd08c11df2cee42 100644 --- a/core/src/main/java/com/gitee/kooder/code/FileRepositoryProvider.java +++ b/core/src/main/java/com/gitee/kooder/code/FileRepositoryProvider.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.code; import com.gitee.kooder.models.CodeRepository; diff --git a/core/src/main/java/com/gitee/kooder/code/FileTraveler.java b/core/src/main/java/com/gitee/kooder/code/FileTraveler.java index c184964a5f756173981f07fe2507658f2200929d..90eaa13acbbb73927a9db7f4df49f9841200d5b5 100644 --- a/core/src/main/java/com/gitee/kooder/code/FileTraveler.java +++ b/core/src/main/java/com/gitee/kooder/code/FileTraveler.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.code; import com.gitee.kooder.models.SourceFile; diff --git a/core/src/main/java/com/gitee/kooder/code/GitRepositoryProvider.java b/core/src/main/java/com/gitee/kooder/code/GitRepositoryProvider.java index 389c77913d7be91e4bb619f7f430baaac0d98a08..eedfb1c2ce0045fb5010ec71be7c82d5d58776da 100644 --- a/core/src/main/java/com/gitee/kooder/code/GitRepositoryProvider.java +++ b/core/src/main/java/com/gitee/kooder/code/GitRepositoryProvider.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.code; import com.gitee.kooder.core.GiteeSearchConfig; @@ -176,6 +191,7 @@ public class GitRepositoryProvider implements RepositoryProvider { for (DiffEntry entry : entries) { if (entry.getChangeType() == DiffEntry.ChangeType.DELETE && traveler != null) { SourceFile doc = new SourceFile(repo.getId(), repo.getName(), entry.getOldPath()); + doc.setEnterprise(repo.getEnterprise()); traveler.deleteDocument(doc); } else { addFileToDocument(repo, git, entry.getNewPath(), entry.getNewId().toObjectId(), traveler); @@ -330,6 +346,7 @@ public class GitRepositoryProvider implements RepositoryProvider { String contents = String.join("\n", codeLines); SourceFile doc = new SourceFile(); + doc.setEnterprise(repo.getEnterprise()); doc.setRepository(new Relation(repo.getId(), repo.getName(), repo.getUrl())); doc.setBranch(git.getRepository().getBranch()); doc.setName(FilenameUtils.getName(path)); //文件名 @@ -337,7 +354,7 @@ public class GitRepositoryProvider implements RepositoryProvider { doc.setLanguage(FileClassifier.languageGuess(path, contents)); //语言 doc.setContents(contents); //源码 doc.setCodeOwner(getCodeOwner(git, path)); //开发者 TODO 如何能支持多个开发者 - var slocCount = slocCounter.countStats(contents, doc.getLanguage()); + SlocCounter.SlocCount slocCount = slocCounter.countStats(contents, doc.getLanguage()); doc.setLines(slocCount.linesCount); //代码行统计 doc.setCommentLines(slocCount.commentCount); doc.setBlankLines(slocCount.blankCount); @@ -365,6 +382,7 @@ public class GitRepositoryProvider implements RepositoryProvider { private SourceFile buildBinaryDocument(CodeRepository repo, Git git, String path, ObjectId objectId) throws IOException { SourceFile doc = new SourceFile(); + doc.setEnterprise(repo.getEnterprise()); doc.setRepository(new Relation(repo.getId(), repo.getName(), repo.getUrl())); doc.setBranch(git.getRepository().getBranch()); doc.setName(FilenameUtils.getName(path)); //文件名 diff --git a/core/src/main/java/com/gitee/kooder/code/RepositoryFactory.java b/core/src/main/java/com/gitee/kooder/code/RepositoryFactory.java index 8423ed4ed08fa26f79e0b40dcfceac63e81c32fa..e0874c9cf87b7a7727496217db619d54e6ce4c12 100644 --- a/core/src/main/java/com/gitee/kooder/code/RepositoryFactory.java +++ b/core/src/main/java/com/gitee/kooder/code/RepositoryFactory.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.code; import com.gitee.kooder.models.CodeRepository; @@ -14,7 +29,7 @@ public class RepositoryFactory { private final static Logger log = LoggerFactory.getLogger(RepositoryFactory.class); - private final static Map providers = new HashMap<>(){{ + private final static Map providers = new HashMap(){{ put(CodeRepository.SCM_GIT, new GitRepositoryProvider()); put(CodeRepository.SCM_SVN, new SvnRepositoryProvider()); put(CodeRepository.SCM_FILE, new FileRepositoryProvider()); diff --git a/core/src/main/java/com/gitee/kooder/code/RepositoryManager.java b/core/src/main/java/com/gitee/kooder/code/RepositoryManager.java index 31c908de72f95cfaceaebcf96aac2cf95ad97956..41b9b886c5f73f07e31170d130cf41db4e100c8a 100644 --- a/core/src/main/java/com/gitee/kooder/code/RepositoryManager.java +++ b/core/src/main/java/com/gitee/kooder/code/RepositoryManager.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.code; import com.gitee.kooder.core.Constants; diff --git a/core/src/main/java/com/gitee/kooder/code/RepositoryProvider.java b/core/src/main/java/com/gitee/kooder/code/RepositoryProvider.java index 61b60369390ec3c8866ba07b4faf28029d6048d3..da190090faab8b9e9ca44b8521b3408e8987f2ca 100644 --- a/core/src/main/java/com/gitee/kooder/code/RepositoryProvider.java +++ b/core/src/main/java/com/gitee/kooder/code/RepositoryProvider.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.code; import com.gitee.kooder.models.CodeRepository; diff --git a/core/src/main/java/com/gitee/kooder/code/SvnRepositoryProvider.java b/core/src/main/java/com/gitee/kooder/code/SvnRepositoryProvider.java index 4a3d80eba1bca9bee13eec1a2cf0b669b4a71f62..27f7abefeab3429f32899124b2f51c7f0347e2fe 100644 --- a/core/src/main/java/com/gitee/kooder/code/SvnRepositoryProvider.java +++ b/core/src/main/java/com/gitee/kooder/code/SvnRepositoryProvider.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.code; import com.gitee.kooder.models.CodeRepository; diff --git a/core/src/main/java/com/gitee/kooder/code/TechCodeAnalyzer.java b/core/src/main/java/com/gitee/kooder/code/TechCodeAnalyzer.java index 71cb62b90e61fa410da60545d5e7dbd2c53c5d6b..73b348ad49af1ba44d4a2e5be10efd5f96da8b0e 100644 --- a/core/src/main/java/com/gitee/kooder/code/TechCodeAnalyzer.java +++ b/core/src/main/java/com/gitee/kooder/code/TechCodeAnalyzer.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.code; import org.apache.lucene.analysis.*; diff --git a/core/src/main/java/com/gitee/kooder/code/TechCodeTokenizer.java b/core/src/main/java/com/gitee/kooder/code/TechCodeTokenizer.java index 0096c4e0ad4d4ada3f0c7aec3bfb9ae6fca8a4a7..8dabf6ee8eb29db501282c32af3f1c72cda4e1e0 100644 --- a/core/src/main/java/com/gitee/kooder/code/TechCodeTokenizer.java +++ b/core/src/main/java/com/gitee/kooder/code/TechCodeTokenizer.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.code; import org.apache.lucene.analysis.Tokenizer; @@ -20,7 +35,7 @@ public final class TechCodeTokenizer extends Tokenizer { public final static char[] beginTokens = {'+','#'}; public final static char[] endTokens = {'.'}; - public final static Map synonymWords = new HashMap<>(){{ + public final static Map synonymWords = new HashMap(){{ put(".net", new String[]{"dotnet"}); put("c#", new String[]{"csharp"}); put("c++", new String[]{"cpp"}); @@ -134,7 +149,7 @@ public final class TechCodeTokenizer extends Tokenizer { if(ew.word.indexOf('.', 1)>0) { String[] pics = ew.word.split("\\."); for(int i=0;i + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.core; import com.gitee.kooder.code.TechCodeAnalyzer; diff --git a/core/src/main/java/com/gitee/kooder/core/Configuration.java b/core/src/main/java/com/gitee/kooder/core/Configuration.java index c7ae8b1e2f4f1028e11322b4934ac234dcf7d193..259b598b14c029ae8389a9803d1674f7753ad684 100644 --- a/core/src/main/java/com/gitee/kooder/core/Configuration.java +++ b/core/src/main/java/com/gitee/kooder/core/Configuration.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.core; import org.apache.commons.lang3.math.NumberUtils; diff --git a/core/src/main/java/com/gitee/kooder/core/Constants.java b/core/src/main/java/com/gitee/kooder/core/Constants.java index 8fdc5e771ebb385f188a9722d43bf9def13ed5ae..9a547526c4972b01faee67253a450523c46bc2d3 100644 --- a/core/src/main/java/com/gitee/kooder/core/Constants.java +++ b/core/src/main/java/com/gitee/kooder/core/Constants.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.core; /** diff --git a/core/src/main/java/com/gitee/kooder/core/GiteeSearchConfig.java b/core/src/main/java/com/gitee/kooder/core/GiteeSearchConfig.java index 6ce5b7308cc0d0a27c947090debd874a714d92bd..0eee2739b964425f1b5c965dbfd9c55d9a4327a7 100644 --- a/core/src/main/java/com/gitee/kooder/core/GiteeSearchConfig.java +++ b/core/src/main/java/com/gitee/kooder/core/GiteeSearchConfig.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.core; import org.slf4j.Logger; diff --git a/core/src/main/java/com/gitee/kooder/core/SearchHelper.java b/core/src/main/java/com/gitee/kooder/core/SearchHelper.java index 84afff6de7d175858a2f63c712f343c141ecd310..43d7b098a6d80526f69bde065a764e3de8f51c29 100644 --- a/core/src/main/java/com/gitee/kooder/core/SearchHelper.java +++ b/core/src/main/java/com/gitee/kooder/core/SearchHelper.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.core; import com.gitee.kooder.models.CodeLine; diff --git a/core/src/main/java/com/gitee/kooder/core/SearchKey.java b/core/src/main/java/com/gitee/kooder/core/SearchKey.java index fc807fc5def0b9b491a7e4927fb6feca0c796960..d5fc19ebc595a789261bbff665ceb1a799ed5367 100644 --- a/core/src/main/java/com/gitee/kooder/core/SearchKey.java +++ b/core/src/main/java/com/gitee/kooder/core/SearchKey.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.core; import java.io.BufferedReader; diff --git a/core/src/main/java/com/gitee/kooder/exception/GiteeException.java b/core/src/main/java/com/gitee/kooder/exception/GiteeException.java deleted file mode 100644 index 252a3081028e3ba7e73663c8219904e0360c8906..0000000000000000000000000000000000000000 --- a/core/src/main/java/com/gitee/kooder/exception/GiteeException.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.gitee.kooder.exception; - -/** - * Gitee related exception - * - * @author zhanggx - */ -public class GiteeException extends Exception { - - public GiteeException(String message) { - super(message); - } - -} diff --git a/core/src/main/java/com/gitee/kooder/index/IndexException.java b/core/src/main/java/com/gitee/kooder/index/IndexException.java index 9ff3df4c1a853a3207d64a18bdf766bd79c476e7..94a2b863fcea15d9e2c09001ade441de095c5777 100644 --- a/core/src/main/java/com/gitee/kooder/index/IndexException.java +++ b/core/src/main/java/com/gitee/kooder/index/IndexException.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.index; /** diff --git a/core/src/main/java/com/gitee/kooder/index/IndexManager.java b/core/src/main/java/com/gitee/kooder/index/IndexManager.java index 7acd0514cb0a3dc946f8e881710eba62d6e690a6..442cc89685b6b923ccd0b664d185c0b731a2e3aa 100644 --- a/core/src/main/java/com/gitee/kooder/index/IndexManager.java +++ b/core/src/main/java/com/gitee/kooder/index/IndexManager.java @@ -1,9 +1,28 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.index; import com.gitee.kooder.core.Constants; +import com.gitee.kooder.models.Repository; +import com.gitee.kooder.models.Searchable; import com.gitee.kooder.queue.QueueTask; import com.gitee.kooder.storage.StorageFactory; import org.apache.lucene.document.Document; +import org.apache.lucene.document.LongPoint; +import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.facet.*; import org.apache.lucene.facet.taxonomy.*; import org.apache.lucene.index.*; @@ -14,11 +33,12 @@ import org.slf4j.LoggerFactory; import static com.gitee.kooder.core.Constants.FIELD_ID; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; /** - * 索引管理器 + * kooder index mananger * @author Winter Lau */ public class IndexManager { @@ -71,7 +91,31 @@ public class IndexManager { update(docs, i_writer, t_writer); break; case QueueTask.ACTION_DELETE: - deleteById(task.getObjects().stream().map(o -> o.getId()).collect(Collectors.toList()), i_writer); + List objects = task.getObjects().stream().map(o -> o.getId()).collect(Collectors.toList()); + Query[] queries = objects.stream().map(id -> NumericDocValuesField.newSlowExactQuery(FIELD_ID, id)).toArray(Query[]::new); + i_writer.deleteDocuments(queries); + List repos = new ArrayList<>(); + for(Searchable obj : task.getObjects()) { + if(obj instanceof Repository) { + repos.add(obj.getId()); + } + } + log.info("Documents['{}'] {} deleted.", task.getType(), objects); + // Delete repository need to delete it's related issues and codes + if(repos.size() > 0) { + // Delete issues of this repository + try (IndexWriter issues = StorageFactory.getIndexWriter(Constants.TYPE_ISSUE)) { + Query[] i_querys = repos.stream().map(id -> LongPoint.newExactQuery(Constants.FIELD_REPO_ID, id)).toArray(Query[]::new); + issues.deleteDocuments(i_querys); + log.info("Issues of repositories : {} deleted.", repos); + } + // Delete code repositories + try (IndexWriter codes = StorageFactory.getIndexWriter(Constants.TYPE_CODE)) { + Query[] i_querys = repos.stream().map(id -> LongPoint.newExactQuery(Constants.FIELD_REPO_ID, id)).toArray(Query[]::new); + codes.deleteDocuments(i_querys); + log.info("Codes of repositories : {} deleted.", repos); + } + } } return task.getObjects().size(); } @@ -115,18 +159,6 @@ public class IndexManager { return docs.size(); } - /** - * 删除文档 - * @param docs - * @param i_writer - * @return - * @throws IOException - */ - public static long deleteById(List docs, IndexWriter i_writer) throws IOException { - Query[] queries = docs.stream().map(id -> new TermQuery(new Term(FIELD_ID, String.valueOf(id)))).toArray(Query[]::new); - return i_writer.deleteDocuments(queries); - } - /** * 将普通 document 转成 facet 文档 * @param taxonomyWriter diff --git a/core/src/main/java/com/gitee/kooder/jcseg/JcsegTokenizer.java b/core/src/main/java/com/gitee/kooder/jcseg/JcsegTokenizer.java index fce5bff0c736500791d06ef2ea8413b1f2105bcb..7f158c8d7b8f40aef14db4709c9a99de26610204 100644 --- a/core/src/main/java/com/gitee/kooder/jcseg/JcsegTokenizer.java +++ b/core/src/main/java/com/gitee/kooder/jcseg/JcsegTokenizer.java @@ -32,7 +32,7 @@ import org.lionsoul.jcseg.segmenter.SegmenterConfig; * after invoke the reset, global object input will be available *

* - *

jcseg tokennizer for lucene on or after 5.1.0

+ *

jcseg tokenizer for lucene on or after 5.1.0

* * @author chenxin */ @@ -47,7 +47,7 @@ public class JcsegTokenizer extends Tokenizer /** * field level offset tracker for multiple-value field - * like the Array field in Elasticseach or Solr + * like the Array field in Elasticsearch or Solr */ private int fieldOffset = 0; diff --git a/core/src/main/java/com/gitee/kooder/models/CodeLine.java b/core/src/main/java/com/gitee/kooder/models/CodeLine.java index 2ed7155d396030ebeaccf0bc59af096366a433a2..3233d1da77b2c818ac573bc88173a018af92c775 100644 --- a/core/src/main/java/com/gitee/kooder/models/CodeLine.java +++ b/core/src/main/java/com/gitee/kooder/models/CodeLine.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.models; /** diff --git a/core/src/main/java/com/gitee/kooder/models/CodeOwner.java b/core/src/main/java/com/gitee/kooder/models/CodeOwner.java index d7d97bfb1a4814b15e46c4fd2ad769d78a68ba0a..4b8d8d8760705ad09d109d78858b50b629212fbb 100644 --- a/core/src/main/java/com/gitee/kooder/models/CodeOwner.java +++ b/core/src/main/java/com/gitee/kooder/models/CodeOwner.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.models; /** diff --git a/core/src/main/java/com/gitee/kooder/models/CodeRepository.java b/core/src/main/java/com/gitee/kooder/models/CodeRepository.java index f809148399d6511ac218d8fea8e6cee200d7eb47..03813939d610d5f2f1815ad890ba954208c182c2 100644 --- a/core/src/main/java/com/gitee/kooder/models/CodeRepository.java +++ b/core/src/main/java/com/gitee/kooder/models/CodeRepository.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.models; import com.gitee.kooder.code.RepositoryManager; @@ -21,6 +36,7 @@ public class CodeRepository extends Searchable { public final static String STATUS_DONE = "indexed"; public final static String STATUS_FETCH = "fetched"; + private int enterprise = 0; //所属企业id private String scm; //代码源类型:git/svn/file private String name; //仓库名称 private String url; //仓库地址,ex: https://gitee.com/ld/J2Cache @@ -30,14 +46,11 @@ public class CodeRepository extends Searchable { public CodeRepository(){} - public CodeRepository(long id) { - super.setId(id); - } - public CodeRepository(Repository r) { this.id = r.getId(); this.name = r.getName(); this.url = r.getUrl(); + this.enterprise = (int)r.getEnterprise().getId(); } /** @@ -96,6 +109,14 @@ public class CodeRepository extends Searchable { this.status = status; } + public int getEnterprise() { + return enterprise; + } + + public void setEnterprise(int enterprise) { + this.enterprise = enterprise; + } + /** * generate lucene document * @@ -105,6 +126,7 @@ public class CodeRepository extends Searchable { public Document getDocument() { Document doc = new Document(); doc.add(new StringField(Constants.FIELD_REPO_ID, this.getIdAsString(), Field.Store.YES)); + doc.add(new StoredField(Constants.FIELD_ENTERPRISE_ID, this.getEnterprise())); doc.add(new StoredField(Constants.FIELD_REPO_URL, this.getUrl())); doc.add(new StoredField(Constants.FIELD_REPO_NAME, this.getName())); if(this.getLastCommitId() != null) @@ -125,6 +147,7 @@ public class CodeRepository extends Searchable { @Override public CodeRepository setDocument(Document doc) { this.setId(NumberUtils.toLong(doc.get(Constants.FIELD_REPO_ID), 0)); + this.setEnterprise(NumberUtils.toInt(doc.get(Constants.FIELD_ENTERPRISE_ID), 0)); this.setName(doc.get(Constants.FIELD_REPO_NAME)); this.setUrl(doc.get(Constants.FIELD_REPO_URL)); this.setLastCommitId(doc.get(Constants.FIELD_REVISION)); diff --git a/core/src/main/java/com/gitee/kooder/models/GitlabProjectEvent.java b/core/src/main/java/com/gitee/kooder/models/GitlabProjectEvent.java deleted file mode 100644 index d16babc30f87218878455760f268f400cf299310..0000000000000000000000000000000000000000 --- a/core/src/main/java/com/gitee/kooder/models/GitlabProjectEvent.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.gitee.kooder.models; - -/** - * 仓库事件 - * @author Winter Lau - */ -public class GitlabProjectEvent { - - public final static String E_PROJECT_CREATE = "project_create"; - public final static String E_PROJECT_DESTROY = "project_destroy"; - public final static String E_PROJECT_RENAME = "project_rename"; - public final static String E_PROJECT_TRANSFER = "project_transfer"; - public final static String E_PROJECT_UPDATE = "project_update"; - -} diff --git a/core/src/main/java/com/gitee/kooder/models/Issue.java b/core/src/main/java/com/gitee/kooder/models/Issue.java index 5dadcfefe615427738e8377deea984f870ca679f..e7c3a94b77b7c42e9f452c987f8faf6aed4a5f1f 100644 --- a/core/src/main/java/com/gitee/kooder/models/Issue.java +++ b/core/src/main/java/com/gitee/kooder/models/Issue.java @@ -1,5 +1,21 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.models; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.gitee.kooder.core.Constants; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.math.NumberUtils; @@ -22,10 +38,10 @@ public final class Issue extends Searchable { public final static int STATE_REOPENED = 0x03; protected String ident; - protected Relation enterprise = Relation.EMPTY; - protected Relation project = Relation.EMPTY; - protected Relation repository = Relation.EMPTY; - protected Relation owner = Relation.EMPTY ; + protected Relation enterprise = Relation.EMPTY(); + protected Relation project = Relation.EMPTY(); + protected Relation repository = Relation.EMPTY(); + protected Relation owner = Relation.EMPTY(); protected String title; protected String description; @@ -43,8 +59,8 @@ public final class Issue extends Searchable { public Issue(org.gitlab4j.api.models.Issue issue) { this.id = issue.getId(); this.ident = issue.getProjectId() + "_" + issue.getId(); - this.enterprise = Relation.EMPTY; - this.project = Relation.EMPTY; + //this.enterprise = Relation.EMPTY(); + //this.project = Relation.EMPTY(); this.repository = new Relation(issue.getProjectId(), null, null); this.owner = new Relation(issue.getAuthor().getId(), issue.getAuthor().getName(), issue.getAuthor().getWebUrl()); this.title = issue.getTitle(); @@ -60,8 +76,8 @@ public final class Issue extends Searchable { public Issue(IssueEvent e) { this.id = e.getObjectAttributes().getId(); this.ident = e.getProject().getId() + "_" + this.id; - this.enterprise = Relation.EMPTY; - this.project = Relation.EMPTY; + //this.enterprise = Relation.EMPTY(); + //this.project = Relation.EMPTY(); this.repository = new Relation(e.getProject().getId(), e.getProject().getName(), e.getProject().getUrl()); this.owner = new Relation(e.getObjectAttributes().getAuthorId(), e.getUser().getName(), e.getUser().getWebUrl()); this.title = e.getObjectAttributes().getTitle(); @@ -79,28 +95,12 @@ public final class Issue extends Searchable { setDocument(doc); } - public Issue(com.gitee.kooder.models.gitee.Issue issue) { - this.id = issue.getId(); - this.ident = issue.getRepository().getId() + "_" + this.id; - this.enterprise = Relation.EMPTY; - this.project = Relation.EMPTY; - this.repository = new Relation(issue.getRepository().getId(), issue.getRepository().getName(), issue.getRepository().getUrl()); - this.owner = new Relation(issue.getUser().getId(), issue.getUser().getName(), issue.getUser().getHtmlUrl()); - this.title = issue.getTitle(); - this.description = issue.getBody(); - this.url = issue.getHtmlUrl(); - this.labels = new ArrayList<>(issue.getLabels()); - this.createdAt = issue.getCreatedAt().getTime(); - this.updatedAt = issue.getUpdatedAt().getTime(); - this.state = com.gitee.kooder.models.gitee.Issue.STATE_OPEN.equals(issue.getState()) - || com.gitee.kooder.models.gitee.Issue.STATE_PROGRESSING.equals(issue.getState()) ? STATE_OPENED : STATE_CLOSED; - } - /** * Read fields from document * @param doc */ @Override + @JsonIgnore public Issue setDocument(Document doc) { this.id = NumberUtils.toInt(doc.get(Constants.FIELD_ID), 0); this.ident = doc.get(Constants.FIELD_IDENT); @@ -130,6 +130,7 @@ public final class Issue extends Searchable { * @return */ @Override + @JsonIgnore public Document getDocument() { Document doc = super.newDocument(); @@ -139,38 +140,26 @@ public final class Issue extends Searchable { doc.add(new StoredField(Constants.FIELD_URL, url)); doc.add(new TextField(Constants.FIELD_TAGS, String.join("\n", labels), Field.Store.NO)); - doc.add(new NumericDocValuesField(Constants.FIELD_CREATED_AT, createdAt)); - doc.add(new StoredField(Constants.FIELD_CREATED_AT, createdAt)); - doc.add(new NumericDocValuesField(Constants.FIELD_UPDATED_AT, updatedAt)); - doc.add(new StoredField(Constants.FIELD_UPDATED_AT, updatedAt)); - doc.add(new NumericDocValuesField(Constants.FIELD_CLOSED_AT, closedAt)); - doc.add(new StoredField(Constants.FIELD_CLOSED_AT, closedAt)); - - doc.add(new IntPoint(Constants.FIELD_BLOCK, block)); - doc.add(new StoredField(Constants.FIELD_BLOCK, block)); - - doc.add(new IntPoint(Constants.FIELD_VISIBILITY, visibility)); - doc.add(new StoredField(Constants.FIELD_VISIBILITY, visibility)); + super.addNumToDoc(doc, Constants.FIELD_CREATED_AT, createdAt); + super.addNumToDoc(doc, Constants.FIELD_UPDATED_AT, updatedAt); + super.addNumToDoc(doc, Constants.FIELD_CLOSED_AT, closedAt); - doc.add(new IntPoint(Constants.FIELD_STATUS, state)); - doc.add(new StoredField(Constants.FIELD_STATUS, state)); + super.addIntToDoc(doc, Constants.FIELD_BLOCK, block); + super.addIntToDoc(doc, Constants.FIELD_VISIBILITY, visibility); + super.addIntToDoc(doc, Constants.FIELD_STATUS, state); //enterprise info (just for gitee) enterprise: - doc.add(new LongPoint(Constants.FIELD_ENTERPRISE_ID, this.enterprise.id)); - doc.add(new StoredField(Constants.FIELD_ENTERPRISE_ID, enterprise.id)); + super.addLongToDoc(doc, Constants.FIELD_ENTERPRISE_ID, this.enterprise.id); //program info (just for gitee) program: - doc.add(new LongPoint(Constants.FIELD_PROGRAM_ID, this.project.id)); - doc.add(new StoredField(Constants.FIELD_PROGRAM_ID, project.id)); + super.addLongToDoc(doc, Constants.FIELD_PROGRAM_ID, this.project.id); //repository info (just for gitee) repository: - doc.add(new LongPoint(Constants.FIELD_REPO_ID, this.repository.id)); - doc.add(new StoredField(Constants.FIELD_REPO_ID, repository.id)); + super.addLongToDoc(doc, Constants.FIELD_REPO_ID, this.repository.id); //owner info owner: - doc.add(new LongPoint(Constants.FIELD_USER_ID, this.owner.id)); - doc.add(new StoredField(Constants.FIELD_USER_ID, owner.id)); + super.addLongToDoc(doc, Constants.FIELD_USER_ID, this.owner.id); return doc; } diff --git a/core/src/main/java/com/gitee/kooder/models/QueryResult.java b/core/src/main/java/com/gitee/kooder/models/QueryResult.java index f342dcd6dc19f7df514f29e983b09e455838e95d..38165879c9342fd2caab66927691b3d4207b5597 100644 --- a/core/src/main/java/com/gitee/kooder/models/QueryResult.java +++ b/core/src/main/java/com/gitee/kooder/models/QueryResult.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.models; import com.gitee.kooder.core.Constants; @@ -43,21 +58,21 @@ public class QueryResult { case Constants.TYPE_REPOSITORY: Repository repo = new Repository(); repo.setDocument(doc); - repo.set_id(doc_score.doc); - repo.set_score(doc_score.score); + repo.set_doc_id(doc_score.doc); + repo.set_doc_score(doc_score.score); addObject(repo); break; case Constants.TYPE_ISSUE: Issue issue = new Issue(doc); - issue.set_id(doc_score.doc); - issue.set_score(doc_score.score); + issue.set_doc_id(doc_score.doc); + issue.set_doc_score(doc_score.score); addObject(issue); break; case Constants.TYPE_CODE: SourceFile file = new SourceFile(); file.setDocument(doc); - file.set_id(doc_score.doc); - file.set_score(doc_score.score); + file.set_doc_id(doc_score.doc); + file.set_doc_score(doc_score.score); addObject(file); } } diff --git a/core/src/main/java/com/gitee/kooder/models/Relation.java b/core/src/main/java/com/gitee/kooder/models/Relation.java index b9291755af0b93ad0389ce81fc660c6591e4602e..4c8248b02f5e80530aa09a773f8be002fbefe47f 100644 --- a/core/src/main/java/com/gitee/kooder/models/Relation.java +++ b/core/src/main/java/com/gitee/kooder/models/Relation.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.models; /** @@ -6,8 +21,6 @@ package com.gitee.kooder.models; */ public final class Relation { - public final static Relation EMPTY = new Relation(0, "NONE", null); - protected long id; protected String name; protected String url; @@ -20,6 +33,10 @@ public final class Relation { this.url = url; } + public final static Relation EMPTY() { + return new Relation(0, "NONE", null); + } + public long getId() { return id; } @@ -46,6 +63,10 @@ public final class Relation { @Override public boolean equals(Object obj) { + if(obj == null) + return false; + if(!(obj instanceof Relation)) + return false; return id == ((Relation)obj).id; } } diff --git a/core/src/main/java/com/gitee/kooder/models/Repository.java b/core/src/main/java/com/gitee/kooder/models/Repository.java index 8fbba9e46366a422176fb2d0d28fc1e341d2c1d4..df3f0ee087baec1906593bcd23edb223259a030f 100644 --- a/core/src/main/java/com/gitee/kooder/models/Repository.java +++ b/core/src/main/java/com/gitee/kooder/models/Repository.java @@ -1,14 +1,28 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.models; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.gitee.kooder.core.Constants; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.math.NumberUtils; import org.apache.lucene.document.*; -import org.apache.lucene.facet.FacetField; import org.gitlab4j.api.models.Project; import org.gitlab4j.api.models.Visibility; -import java.util.Collections; import java.util.List; /** @@ -21,9 +35,9 @@ public final class Repository extends Searchable { protected String displayName; //nameWithNamespace protected String description; protected String url; - protected Relation enterprise = Relation.EMPTY; - protected Relation project = Relation.EMPTY; - protected Relation owner = Relation.EMPTY; + protected Relation enterprise = Relation.EMPTY(); + protected Relation project = Relation.EMPTY(); + protected Relation owner = Relation.EMPTY(); protected int recomm; //推荐级别 protected int gindex; // Gitee Index protected int block; //是否屏蔽 1屏蔽,0不屏蔽 @@ -51,8 +65,8 @@ public final class Repository extends Searchable { this.name = p.getName(); this.description = p.getDescription(); this.url = p.getHttpUrlToRepo(); - this.enterprise = Relation.EMPTY; - this.project = Relation.EMPTY; + //this.enterprise = Relation.EMPTY(); + //this.project = Relation.EMPTY(); if(p.getOwner() != null) this.owner = new Relation(p.getOwner().getId(), p.getOwner().getName(), p.getOwner().getWebUrl()); this.setVisibility(p.getVisibility()); @@ -69,32 +83,12 @@ public final class Repository extends Searchable { this.setBlock(Constants.REPO_BLOCK_NO); } - public Repository(com.gitee.kooder.models.gitee.Repository repository) { - this.id = repository.getId(); - this.name = repository.getName(); - this.description = repository.getDescription(); - this.url = repository.getHtmlUrl(); - this.enterprise = Relation.EMPTY; - this.project = Relation.EMPTY; - this.owner = new Relation(repository.getOwner().getId(), repository.getOwner().getName(), repository.getOwner().getHtmlUrl()); - this.visibility = repository.getPrivate() ? Constants.VISIBILITY_PRIVATE : repository.getInternal() ? Constants.VISIBILITY_INTERNAL : Constants.VISIBILITY_PUBLIC; - this.license = repository.getLicense(); - this.lang = repository.getLanguage(); - this.readme = ""; - this.fork = 0; - this.tags = Collections.emptyList(); - this.starsCount = repository.getStargazersCount(); - this.forksCount = repository.getForksCount(); - this.createdAt = repository.getCreatedAt().getTime(); - this.updatedAt = repository.getUpdatedAt().getTime(); - this.setBlock(Constants.REPO_BLOCK_NO); - } - /** * generate lucene document * @return */ @Override + @JsonIgnore public Document getDocument() { Document doc = super.newDocument(); doc.add(new TextField(Constants.FIELD_NAME, this.getName(), Field.Store.YES)); @@ -105,43 +99,26 @@ public final class Repository extends Searchable { if(StringUtils.isNotBlank(this.getUrl())) doc.add(new StoredField(Constants.FIELD_URL, this.getUrl())); - doc.add(new NumericDocValuesField(Constants.FIELD_RECOMM, recomm)); - doc.add(new StoredField(Constants.FIELD_RECOMM, recomm)); + super.addNumToDoc(doc, Constants.FIELD_RECOMM, recomm); + super.addNumToDoc(doc, Constants.FIELD_G_INDEX, gindex); + super.addNumToDoc(doc, Constants.FIELD_BLOCK, block); + super.addIntToDoc(doc, Constants.FIELD_VISIBILITY, visibility); - doc.add(new NumericDocValuesField(Constants.FIELD_G_INDEX, gindex)); - doc.add(new StoredField(Constants.FIELD_G_INDEX, gindex)); - - doc.add(new IntPoint(Constants.FIELD_BLOCK, block)); - doc.add(new StoredField(Constants.FIELD_BLOCK, block)); - - doc.add(new IntPoint(Constants.FIELD_VISIBILITY, visibility)); - doc.add(new StoredField(Constants.FIELD_VISIBILITY, visibility)); - - if(StringUtils.isNotBlank(license)) { - doc.add(new FacetField(Constants.FIELD_LICENSE, license)); - doc.add(new StringField(Constants.FIELD_LICENSE, license, Field.Store.YES)); - } + if(StringUtils.isNotBlank(license)) + super.addFacetToDoc(doc, Constants.FIELD_LICENSE, license); - if(StringUtils.isNotBlank(lang)) { - doc.add(new FacetField(Constants.FIELD_LANGUAGE, lang)); - doc.add(new StringField(Constants.FIELD_LANGUAGE, lang, Field.Store.YES)); - } + if(StringUtils.isNotBlank(lang)) + super.addFacetToDoc(doc, Constants.FIELD_LANGUAGE, lang); if(StringUtils.isNotBlank(readme)) doc.add(new TextField(Constants.FIELD_README, readme, Field.Store.NO)); - doc.add(new LongPoint(Constants.FIELD_FORK, fork)); - doc.add(new StoredField(Constants.FIELD_FORK, fork)); - - doc.add(new NumericDocValuesField(Constants.FIELD_STAR_COUNT, starsCount)); - doc.add(new StoredField(Constants.FIELD_STAR_COUNT, starsCount)); - doc.add(new NumericDocValuesField(Constants.FIELD_FORK_COUNT, forksCount)); - doc.add(new StoredField(Constants.FIELD_FORK_COUNT, forksCount)); - doc.add(new NumericDocValuesField(Constants.FIELD_CREATED_AT, createdAt)); - doc.add(new StoredField(Constants.FIELD_CREATED_AT, createdAt)); - doc.add(new NumericDocValuesField(Constants.FIELD_UPDATED_AT, updatedAt)); - doc.add(new StoredField(Constants.FIELD_UPDATED_AT, updatedAt)); + super.addLongToDoc(doc, Constants.FIELD_FORK, fork); + super.addNumToDoc(doc, Constants.FIELD_STAR_COUNT, starsCount); + super.addNumToDoc(doc, Constants.FIELD_FORK_COUNT, forksCount); + super.addNumToDoc(doc, Constants.FIELD_CREATED_AT, createdAt); + super.addNumToDoc(doc, Constants.FIELD_UPDATED_AT, updatedAt); //tags if(tags != null) @@ -152,32 +129,27 @@ public final class Repository extends Searchable { //enterprise info (just for gitee) enterprise: - doc.add(new LongPoint(Constants.FIELD_ENTERPRISE_ID, this.enterprise.id)); - doc.add(new StringField(Constants.FIELD_ENTERPRISE_ID, String.valueOf(enterprise.id), Field.Store.YES)); - if(StringUtils.isNotBlank(enterprise.name)) { - doc.add(new FacetField(Constants.FIELD_ENTERPRISE_NAME, enterprise.name)); - doc.add(new TextField(Constants.FIELD_ENTERPRISE_NAME, enterprise.name, Field.Store.YES)); - } + super.addLongToDoc(doc, Constants.FIELD_ENTERPRISE_ID, this.enterprise.id); + if(StringUtils.isNotBlank(enterprise.name)) + super.addFacetToDoc(doc, Constants.FIELD_ENTERPRISE_NAME, enterprise.name); + if(StringUtils.isNotBlank(enterprise.url)) doc.add(new StoredField(Constants.FIELD_ENTERPRISE_URL, enterprise.url)); //program info (just for gitee) program: - doc.add(new LongPoint(Constants.FIELD_PROGRAM_ID, this.project.id)); - doc.add(new StringField(Constants.FIELD_PROGRAM_ID, String.valueOf(project.id), Field.Store.YES)); - if(StringUtils.isNotBlank(project.name)) { - doc.add(new FacetField(Constants.FIELD_PROGRAM_NAME, project.name)); - doc.add(new TextField(Constants.FIELD_PROGRAM_NAME, project.name, Field.Store.YES)); - } + super.addLongToDoc(doc, Constants.FIELD_PROGRAM_ID, this.project.id); + if(StringUtils.isNotBlank(project.name)) + super.addFacetToDoc(doc, Constants.FIELD_PROGRAM_NAME, project.name); + if(StringUtils.isNotBlank(project.url)) doc.add(new StoredField(Constants.FIELD_PROGRAM_URL, project.url)); //owner info owner: - doc.add(new LongPoint(Constants.FIELD_USER_ID, this.owner.id)); - doc.add(new StringField(Constants.FIELD_USER_ID, String.valueOf(owner.id), Field.Store.YES)); - if(StringUtils.isNotBlank(owner.name)) { - doc.add(new FacetField(Constants.FIELD_USER_NAME, owner.name)); - doc.add(new TextField(Constants.FIELD_USER_NAME, owner.name, Field.Store.YES)); - } + super.addLongToDoc(doc, Constants.FIELD_USER_ID, this.owner.id); + + if(StringUtils.isNotBlank(owner.name)) + super.addFacetToDoc(doc, Constants.FIELD_USER_NAME, owner.name); + if(StringUtils.isNotBlank(owner.url)) doc.add(new StoredField(Constants.FIELD_USER_URL, owner.url)); @@ -190,6 +162,7 @@ public final class Repository extends Searchable { * @param doc */ @Override + @JsonIgnore public Repository setDocument(Document doc) { this.id = NumberUtils.toInt(doc.get(Constants.FIELD_ID), 0); this.name = doc.get(Constants.FIELD_NAME); diff --git a/core/src/main/java/com/gitee/kooder/models/Searchable.java b/core/src/main/java/com/gitee/kooder/models/Searchable.java index 7c4d606c49db29d914e5206a72352664e92037e9..b24c7443ca3677c7fd8baa66a643c7cb1604b5fa 100644 --- a/core/src/main/java/com/gitee/kooder/models/Searchable.java +++ b/core/src/main/java/com/gitee/kooder/models/Searchable.java @@ -1,8 +1,24 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.models; import com.gitee.kooder.core.Constants; import org.apache.commons.lang3.math.NumberUtils; import org.apache.lucene.document.*; +import org.apache.lucene.facet.FacetField; import java.io.Serializable; @@ -13,8 +29,8 @@ import java.io.Serializable; public abstract class Searchable implements Serializable { protected long id; // object id , ex: repo id, issue id - protected int _id; // document id - protected float _score; // document score + protected int _doc_id; // document id + protected float _doc_score; // document score public long getId() { return id; @@ -24,20 +40,20 @@ public abstract class Searchable implements Serializable { this.id = id; } - public int get_id() { - return _id; + public int get_doc_id() { + return _doc_id; } - public void set_id(int _id) { - this._id = _id; + public void set_doc_id(int _doc_id) { + this._doc_id = _doc_id; } - public float get_score() { - return _score; + public float get_doc_score() { + return _doc_score; } - public void set_score(float _score) { - this._score = _score; + public void set_doc_score(float _doc_score) { + this._doc_score = _doc_score; } protected Document newDocument() { @@ -71,4 +87,24 @@ public abstract class Searchable implements Serializable { return NumberUtils.toInt(doc.get(fieldName), def); } + protected void addLongToDoc(Document doc, String fn, long fv) { + doc.add(new LongPoint(fn, fv)); + doc.add(new StoredField(fn, String.valueOf(fv))); + } + + protected void addIntToDoc(Document doc, String fn, int fv) { + doc.add(new IntPoint(fn, fv)); + doc.add(new StoredField(fn, String.valueOf(fv))); + } + + protected void addFacetToDoc(Document doc, String fn, String fv) { + doc.add(new FacetField(fn, fv)); + doc.add(new TextField(fn, fv, Field.Store.YES)); + } + + protected void addNumToDoc(Document doc, String fn, long fv) { + doc.add(new NumericDocValuesField(fn, fv)); + doc.add(new StoredField(fn, fv)); + } + } diff --git a/core/src/main/java/com/gitee/kooder/models/SourceFile.java b/core/src/main/java/com/gitee/kooder/models/SourceFile.java index 5428c83d992659f256bdd924c90960beaa6ded61..dfcd4197c4b420db2f476062a5acbb938f892a63 100644 --- a/core/src/main/java/com/gitee/kooder/models/SourceFile.java +++ b/core/src/main/java/com/gitee/kooder/models/SourceFile.java @@ -1,11 +1,28 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.models; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.gitee.kooder.core.Constants; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.math.NumberUtils; import org.apache.lucene.document.*; -import org.apache.lucene.facet.FacetField; + +import java.util.List; /** * Source File Object @@ -14,7 +31,8 @@ import org.apache.lucene.facet.FacetField; public final class SourceFile extends Searchable { private String uuid; // file unique identify - private Relation repository = Relation.EMPTY; //repository, use this field to delete all files of repository + private int enterprise; // enterprise + private Relation repository = Relation.EMPTY(); //repository, use this field to delete all files of repository private String branch; // branch name private String name; // file name @@ -33,6 +51,8 @@ public final class SourceFile extends Searchable { private String revision; // last commit id + private List result; // code lines with keyword highlight + public SourceFile() {} public SourceFile(long repoId, String repoName, String fileLocation) { @@ -59,8 +79,10 @@ public final class SourceFile extends Searchable { * @param doc */ @Override + @JsonIgnore public SourceFile setDocument(Document doc) { this.uuid = doc.get(Constants.FIELD_UUID); + this.enterprise = NumberUtils.toInt(doc.get(Constants.FIELD_ENTERPRISE_ID), 0); this.repository.id = NumberUtils.toInt(doc.get(Constants.FIELD_REPO_ID)); this.repository.name = doc.get(Constants.FIELD_REPO_NAME); this.repository.url = doc.get(Constants.FIELD_REPO_URL); @@ -87,31 +109,30 @@ public final class SourceFile extends Searchable { * @return */ @Override + @JsonIgnore public Document getDocument() { Document document = new Document(); // Uuid is the primary key for documents document.add(new StringField(Constants.FIELD_UUID, this.uuid, Field.Store.YES)); - document.add(new StringField(Constants.FIELD_BRANCH, this.branch, Field.Store.YES)); + if(StringUtils.isNotBlank(this.branch)) + document.add(new StringField(Constants.FIELD_BRANCH, this.branch, Field.Store.YES)); document.add(new StoredField(Constants.FIELD_URL, this.url)); + super.addLongToDoc(document, Constants.FIELD_ENTERPRISE_ID, this.enterprise); + //repository info - document.add(new LongPoint(Constants.FIELD_REPO_ID, this.repository.id)); - document.add(new StoredField(Constants.FIELD_REPO_ID, this.repository.id)); - document.add(new FacetField(Constants.FIELD_REPO_NAME, this.repository.name)); - document.add(new StringField(Constants.FIELD_REPO_NAME, this.repository.name, Field.Store.YES)); + super.addLongToDoc(document, Constants.FIELD_REPO_ID, this.repository.id); + super.addFacetToDoc(document, Constants.FIELD_REPO_NAME, this.repository.name); document.add(new StringField(Constants.FIELD_REPO_URL, this.repository.url, Field.Store.YES)); //file meta - if (StringUtils.isNotBlank(language)) { - document.add(new FacetField(Constants.FIELD_LANGUAGE, this.getLanguage())); - document.add(new StringField(Constants.FIELD_LANGUAGE, this.getLanguage(), Field.Store.YES)); - } - if (StringUtils.isNotBlank(codeOwner)) { - document.add(new FacetField(Constants.FIELD_CODE_OWNER, this.getCodeOwner())); - document.add(new TextField(Constants.FIELD_CODE_OWNER, this.getCodeOwner(), Field.Store.YES)); - } + if (StringUtils.isNotBlank(language)) + super.addFacetToDoc(document, Constants.FIELD_LANGUAGE, this.language); + + if (StringUtils.isNotBlank(codeOwner)) + super.addFacetToDoc(document, Constants.FIELD_CODE_OWNER, this.codeOwner); //file info document.add(new TextField(Constants.FIELD_FILE_NAME, this.getName(), Field.Store.YES)); @@ -134,8 +155,7 @@ public final class SourceFile extends Searchable { // Extra metadata in this case when it was last indexed long indexTime = System.currentTimeMillis(); - document.add(new NumericDocValuesField(Constants.FIELD_LAST_INDEX, indexTime)); - document.add(new StoredField(Constants.FIELD_LAST_INDEX, indexTime)); + super.addNumToDoc(document, Constants.FIELD_LAST_INDEX, indexTime); return document; } @@ -267,4 +287,20 @@ public final class SourceFile extends Searchable { public void setRevision(String revision) { this.revision = revision; } + + public int getEnterprise() { + return enterprise; + } + + public void setEnterprise(int enterprise) { + this.enterprise = enterprise; + } + + public List getResult() { + return result; + } + + public void setResult(List result) { + this.result = result; + } } diff --git a/core/src/main/java/com/gitee/kooder/models/gitee/EnterpriseHook.java b/core/src/main/java/com/gitee/kooder/models/gitee/EnterpriseHook.java deleted file mode 100644 index babd424cc51381da1f09f560a9f6864bcc71e4e6..0000000000000000000000000000000000000000 --- a/core/src/main/java/com/gitee/kooder/models/gitee/EnterpriseHook.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.gitee.kooder.models.gitee; - -/** - * @author zhanggx - */ -public class EnterpriseHook { - - private String url; - - public String getUrl() { - return url; - } - - public void setUrl(String url) { - this.url = url; - } - -} diff --git a/core/src/main/java/com/gitee/kooder/query/CodeQuery.java b/core/src/main/java/com/gitee/kooder/query/CodeQuery.java index d398a329f9dd847eb78f5c0ec365cf1c934be058..dcdf6c840d351303c889041713f91ea4a3760e9a 100644 --- a/core/src/main/java/com/gitee/kooder/query/CodeQuery.java +++ b/core/src/main/java/com/gitee/kooder/query/CodeQuery.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.query; import com.gitee.kooder.core.AnalyzerFactory; diff --git a/core/src/main/java/com/gitee/kooder/query/IQuery.java b/core/src/main/java/com/gitee/kooder/query/IQuery.java index 65cda58c189a5802304281fcc5aed8ab3c7f0591..e356075372ee98c5a52c361a89e5c1e05191f859 100644 --- a/core/src/main/java/com/gitee/kooder/query/IQuery.java +++ b/core/src/main/java/com/gitee/kooder/query/IQuery.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.query; import com.gitee.kooder.models.QueryResult; diff --git a/core/src/main/java/com/gitee/kooder/query/IssueQuery.java b/core/src/main/java/com/gitee/kooder/query/IssueQuery.java index 4f4ecc75e47d336d891484906c58ee8146b3f455..47949c740e889476d1ecdd849e72c71d003acac8 100644 --- a/core/src/main/java/com/gitee/kooder/query/IssueQuery.java +++ b/core/src/main/java/com/gitee/kooder/query/IssueQuery.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.query; import com.gitee.kooder.core.AnalyzerFactory; diff --git a/core/src/main/java/com/gitee/kooder/query/QueryBase.java b/core/src/main/java/com/gitee/kooder/query/QueryBase.java index 2844b4f1af8e5fbf7215fd2508d7a981f295a1bf..91335a830fc9605c1713eaeebb680dbe3a27d14e 100644 --- a/core/src/main/java/com/gitee/kooder/query/QueryBase.java +++ b/core/src/main/java/com/gitee/kooder/query/QueryBase.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.query; import com.gitee.kooder.core.AnalyzerFactory; @@ -8,14 +23,12 @@ import com.gitee.kooder.storage.StorageFactory; import org.apache.commons.lang3.StringUtils; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Document; -import org.apache.lucene.document.IntPoint; -import org.apache.lucene.document.NumericDocValuesField; +import org.apache.lucene.document.LongPoint; import org.apache.lucene.facet.*; import org.apache.lucene.facet.taxonomy.FastTaxonomyFacetCounts; import org.apache.lucene.facet.taxonomy.TaxonomyReader; import org.apache.lucene.index.IndexNotFoundException; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.Term; import org.apache.lucene.queries.function.FunctionScoreQuery; import org.apache.lucene.queryparser.classic.ParseException; import org.apache.lucene.queryparser.classic.QueryParser; @@ -28,7 +41,7 @@ import java.util.*; import java.util.stream.Collectors; /** - * 搜索基类 + * base class of query * @author Winter Lau */ public abstract class QueryBase implements IQuery { @@ -38,14 +51,14 @@ public abstract class QueryBase implements IQuery { public final static FacetsConfig facetsConfig = new FacetsConfig(); private int enterpriseId = 0; // Search in Enterprise - private List repositories = new ArrayList<>(); // Search in repositories + private List repositories = new ArrayList(); // Search in repositories protected String searchKey; // Search Keyword protected boolean parseSearchKey = false; // Escape Search key ? protected String sort; // Sort field name protected int page = 1; // Search result page index protected int pageSize = 20; // Search result page size protected Map facets = new HashMap(); // Search with facets - protected List filters = new ArrayList<>(); // Search filters + protected List filters = new ArrayList(); // Search filters /** * Get max object indexed . @@ -64,7 +77,7 @@ public abstract class QueryBase implements IQuery { } }catch(IndexNotFoundException e) { }catch(Exception e) { - log.error("Failed to get lastest object from index[" + type() + "]", e); + log.error("Failed to get latest object from index[" + type() + "]", e); } return null; } @@ -178,11 +191,12 @@ public abstract class QueryBase implements IQuery { return query; BooleanQuery.Builder fBuilder = new BooleanQuery.Builder(); + if(enterpriseId > 0) - fBuilder.add(new TermQuery(new Term(Constants.FIELD_ENTERPRISE_ID, String.valueOf(this.getEnterpriseId()))), BooleanClause.Occur.FILTER); + fBuilder.add(LongPoint.newExactQuery(Constants.FIELD_ENTERPRISE_ID, this.getEnterpriseId()), BooleanClause.Occur.FILTER); if(repositories.size() > 0) - fBuilder.add(IntPoint.newSetQuery(Constants.FIELD_REPO_ID, repositories), BooleanClause.Occur.FILTER); + fBuilder.add(LongPoint.newSetQuery(Constants.FIELD_REPO_ID, repositories), BooleanClause.Occur.FILTER); for(Query filter : filters) fBuilder.add(filter, BooleanClause.Occur.FILTER); @@ -260,11 +274,11 @@ public abstract class QueryBase implements IQuery { return this; } - public List getRepositories() { + public List getRepositories() { return repositories; } - public QueryBase addRepositories(List repositories) { + public QueryBase addRepositories(List repositories) { this.repositories.addAll(repositories); return this; } diff --git a/core/src/main/java/com/gitee/kooder/query/QueryException.java b/core/src/main/java/com/gitee/kooder/query/QueryException.java index c82d213f904cd2b4a5b5d7fa10c731e93af4c4a3..5242c9a6b176ec83f597b08b0c92833dd2e40430 100644 --- a/core/src/main/java/com/gitee/kooder/query/QueryException.java +++ b/core/src/main/java/com/gitee/kooder/query/QueryException.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.query; /** diff --git a/core/src/main/java/com/gitee/kooder/query/QueryFactory.java b/core/src/main/java/com/gitee/kooder/query/QueryFactory.java index a8978566cf66efa1fc9dbc650690c987670c31dd..0318a03a7504a2904908683c856b93f6ce6eb3fb 100644 --- a/core/src/main/java/com/gitee/kooder/query/QueryFactory.java +++ b/core/src/main/java/com/gitee/kooder/query/QueryFactory.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.query; /** diff --git a/core/src/main/java/com/gitee/kooder/query/RepoQuery.java b/core/src/main/java/com/gitee/kooder/query/RepoQuery.java index 6af6042cf34306e8de1916d6761085198fd9b5c3..16657de8d3021c52d77db2ebfcada729ae19e7a6 100644 --- a/core/src/main/java/com/gitee/kooder/query/RepoQuery.java +++ b/core/src/main/java/com/gitee/kooder/query/RepoQuery.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.query; import com.gitee.kooder.core.AnalyzerFactory; diff --git a/core/src/main/java/com/gitee/kooder/queue/EmbedQueueProvider.java b/core/src/main/java/com/gitee/kooder/queue/EmbedQueueProvider.java index 73f53e608e864a71322dd02aaf75426a2b506a0d..8c964af06e4c4b1e32c0d9b05ec7865d0c538b18 100644 --- a/core/src/main/java/com/gitee/kooder/queue/EmbedQueueProvider.java +++ b/core/src/main/java/com/gitee/kooder/queue/EmbedQueueProvider.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.queue; import com.gitee.kooder.core.GiteeSearchConfig; @@ -26,7 +41,7 @@ public class EmbedQueueProvider implements QueueProvider { int batch_size = NumberUtils.toInt(props.getProperty("embed.batch_size", "10000"), 10000); Path path = checkoutPath(GiteeSearchConfig.getPath(props.getProperty("embed.path"))); - for(String type : types()) { + for(String type : getAllTypes()) { Path typePath = checkoutPath(path.resolve(type)); fileQueues.put(type, FileQueue.batched().name(type) .folder(typePath) diff --git a/core/src/main/java/com/gitee/kooder/queue/Queue.java b/core/src/main/java/com/gitee/kooder/queue/Queue.java index f1fa272052a81379a88880ecd3939ebe88c060cd..af6743a395a6bd27439cf002e04689a22b067424 100644 --- a/core/src/main/java/com/gitee/kooder/queue/Queue.java +++ b/core/src/main/java/com/gitee/kooder/queue/Queue.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.queue; import java.util.Collection; diff --git a/core/src/main/java/com/gitee/kooder/queue/QueueFactory.java b/core/src/main/java/com/gitee/kooder/queue/QueueFactory.java index e61022c5171d74b6133ad43be8ee21d93d62a1a4..dc49429681a1018a1a682432793fc770f1aa7bba 100644 --- a/core/src/main/java/com/gitee/kooder/queue/QueueFactory.java +++ b/core/src/main/java/com/gitee/kooder/queue/QueueFactory.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.queue; import com.gitee.kooder.core.GiteeSearchConfig; diff --git a/core/src/main/java/com/gitee/kooder/queue/QueueProvider.java b/core/src/main/java/com/gitee/kooder/queue/QueueProvider.java index dfba1e63c0b23cc31a3deb2427599d2bf5b3b56d..ed67cb21dd6ee4b2a9db85485638d22adc79ef1b 100644 --- a/core/src/main/java/com/gitee/kooder/queue/QueueProvider.java +++ b/core/src/main/java/com/gitee/kooder/queue/QueueProvider.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.queue; import com.gitee.kooder.core.Constants; @@ -23,7 +38,7 @@ public interface QueueProvider extends AutoCloseable { * 获取支持的所有任务类型 * @return */ - default List types() { + default List getAllTypes() { return TYPES; } diff --git a/core/src/main/java/com/gitee/kooder/queue/QueueTask.java b/core/src/main/java/com/gitee/kooder/queue/QueueTask.java index 8d22eca58e2407779fc55e21c3b28cab27fb2983..2a26ebb37e8241add8c857b893ae53ad09347b44 100644 --- a/core/src/main/java/com/gitee/kooder/queue/QueueTask.java +++ b/core/src/main/java/com/gitee/kooder/queue/QueueTask.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.queue; import java.io.IOException; diff --git a/core/src/main/java/com/gitee/kooder/queue/RedisQueueProvider.java b/core/src/main/java/com/gitee/kooder/queue/RedisQueueProvider.java index ff78910740a6e0b425f06863b4f903165c875453..da90ad90a10d87986159c5635622382f619f8da6 100644 --- a/core/src/main/java/com/gitee/kooder/queue/RedisQueueProvider.java +++ b/core/src/main/java/com/gitee/kooder/queue/RedisQueueProvider.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.queue; import io.lettuce.core.RedisClient; diff --git a/core/src/main/java/com/gitee/kooder/storage/DiskIndexStorage.java b/core/src/main/java/com/gitee/kooder/storage/DiskIndexStorage.java index 2bcac97f9ef89510449f9695b1130135b7ccd15a..2c06195c4358130e7a930e6871d9bc49cc3790dd 100644 --- a/core/src/main/java/com/gitee/kooder/storage/DiskIndexStorage.java +++ b/core/src/main/java/com/gitee/kooder/storage/DiskIndexStorage.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.storage; import com.gitee.kooder.core.AnalyzerFactory; @@ -25,7 +40,7 @@ import java.util.Properties; import static com.gitee.kooder.core.Constants.TYPE_CODE; /** - * 磁盘索引管理器 + * store index in disk * @author Winter Lau */ public class DiskIndexStorage implements IndexStorage { diff --git a/core/src/main/java/com/gitee/kooder/storage/IndexStorage.java b/core/src/main/java/com/gitee/kooder/storage/IndexStorage.java index 9f440c4e0fd6d3f6d11ed229f1a950169c68a7b9..caa1789797c7da888db455d42127441978e0689c 100644 --- a/core/src/main/java/com/gitee/kooder/storage/IndexStorage.java +++ b/core/src/main/java/com/gitee/kooder/storage/IndexStorage.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.storage; import com.gitee.kooder.core.Constants; diff --git a/core/src/main/java/com/gitee/kooder/storage/StorageFactory.java b/core/src/main/java/com/gitee/kooder/storage/StorageFactory.java index 12c493ff659ec760dbf922adc55e191bf3382999..1e3662f9a33b694a96c7f3be097080549199eb88 100644 --- a/core/src/main/java/com/gitee/kooder/storage/StorageFactory.java +++ b/core/src/main/java/com/gitee/kooder/storage/StorageFactory.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.storage; import com.gitee.kooder.core.GiteeSearchConfig; diff --git a/core/src/main/java/com/gitee/kooder/utils/BatchTaskRunner.java b/core/src/main/java/com/gitee/kooder/utils/BatchTaskRunner.java index 9574d366e55ea0023da7fc14ff773344cc1ed684..8628cb09db4783901478d590683b23d2d100a3f6 100644 --- a/core/src/main/java/com/gitee/kooder/utils/BatchTaskRunner.java +++ b/core/src/main/java/com/gitee/kooder/utils/BatchTaskRunner.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.utils; import java.util.List; diff --git a/core/src/main/java/com/gitee/kooder/utils/FileClassifier.java b/core/src/main/java/com/gitee/kooder/utils/FileClassifier.java index 1007a6e2781068677c8148d62379ad03f2a04abe..da9935598202d1a19186c9c2cce174b99a666995 100644 --- a/core/src/main/java/com/gitee/kooder/utils/FileClassifier.java +++ b/core/src/main/java/com/gitee/kooder/utils/FileClassifier.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.utils; import com.fasterxml.jackson.core.type.TypeReference; @@ -19,7 +34,7 @@ public class FileClassifier { static { try (InputStream stream = FileClassifier.class.getResourceAsStream("/languages.json")){ - TypeReference> typeRef = new TypeReference<>(){}; + TypeReference> typeRef = new TypeReference>(){}; database = Collections.unmodifiableMap(JsonUtils.readValue(stream, typeRef)); } catch (IOException ex) { ex.printStackTrace(); @@ -35,15 +50,14 @@ public class FileClassifier { } /** - * Given a filename guesses the file type + * Guesses the file type of given filename */ public static String languageGuess(String fileName, String content) { fileName = fileName.toLowerCase(); - var matches = new ArrayList(); - var extension = ""; + String extension = ""; // Try finding based on full name match - matches = checkIfFilenameExists(fileName); + List matches = checkIfFilenameExists(fileName); // Try finding using the whole name EG LICENSE if (matches.isEmpty()) { @@ -73,11 +87,11 @@ public class FileClassifier { } // We have multiple matches, so try to work out which one is the most likely result - var toSort = new HashMap(); + HashMap toSort = new HashMap(); - for (var m : matches) { + for (String m : matches) { toSort.put(m, 0); - for (var keyword : database.get(m).keywords) { + for (String keyword : database.get(m).keywords) { if (content.contains(keyword)) { toSort.put(m, toSort.get(m) + 1); } @@ -98,13 +112,13 @@ public class FileClassifier { return result; } - private static ArrayList checkIfExtentionExists(String extension) { - var matches = new ArrayList(); + private static List checkIfExtentionExists(String extension) { + List matches = new ArrayList<>(); for (String key : database.keySet()) { - var fileClassifierResult = database.get(key); + FileClassifierResult fileClassifierResult = database.get(key); - for (var ext : fileClassifierResult.extensions) { + for (String ext : fileClassifierResult.extensions) { if (extension.equals(ext)) { matches.add(key); } @@ -114,14 +128,14 @@ public class FileClassifier { return matches; } - private static ArrayList checkIfFilenameExists(String extension) { - var matches = new ArrayList(); + private static List checkIfFilenameExists(String extension) { + List matches = new ArrayList<>(); for (String key : database.keySet()) { - var fileClassifierResult = database.get(key); + FileClassifierResult fileClassifierResult = database.get(key); if (fileClassifierResult.filenames != null) { - for (var ext : fileClassifierResult.filenames) { + for (String ext : fileClassifierResult.filenames) { if (extension.equals(ext)) { matches.add(key); } diff --git a/core/src/main/java/com/gitee/kooder/utils/FileClassifierResult.java b/core/src/main/java/com/gitee/kooder/utils/FileClassifierResult.java index 6810db6e11cb562281f788ecd6143d0d992a81f8..5899eca0f62be6a1c309c1fac1fa6a5a59bb8d0b 100644 --- a/core/src/main/java/com/gitee/kooder/utils/FileClassifierResult.java +++ b/core/src/main/java/com/gitee/kooder/utils/FileClassifierResult.java @@ -1,7 +1,23 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.utils; /** - * 对应 languages.json 中的语言定义 + * This class used to mapping language.json segment defined + * @author Winter Lau */ public class FileClassifierResult { diff --git a/core/src/main/java/com/gitee/kooder/utils/HttpUtils.java b/core/src/main/java/com/gitee/kooder/utils/HttpUtils.java index 7c34c536d5657174e1dd3d3e35965b6ab55dda74..269d13ac6215f49052ffb279e748f1eb45928856 100644 --- a/core/src/main/java/com/gitee/kooder/utils/HttpUtils.java +++ b/core/src/main/java/com/gitee/kooder/utils/HttpUtils.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.utils; import okhttp3.*; @@ -7,6 +22,8 @@ import java.util.Map; import java.util.Objects; /** + * http util by okhttp3 + * * @author zhanggx */ public class HttpUtils { @@ -22,11 +39,27 @@ public class HttpUtils { CLIENT = new OkHttpClient(); } + /** + * http get + * + * @param url + * @param params + * @return + * @throws IOException + */ public static Response get(String url, Map params) throws IOException { Request request = getRequestBuilder(url, params).build(); return getResponse(request); } + /** + * http post application/json + * + * @param url + * @param params + * @return + * @throws IOException + */ public static Response postJson(String url, Map params) throws IOException { Request request = getRequestBuilder(url) .post(RequestBody.create(JsonUtils.toJson(params), APPLICATION_JSON_UTF8)) diff --git a/core/src/main/java/com/gitee/kooder/utils/JsonUtils.java b/core/src/main/java/com/gitee/kooder/utils/JsonUtils.java index e2fc7a7b6570d9f32586364c2cdc4a8a0b128f14..ac4668069d34d89b93f11bd164a9281ef5db4b7e 100644 --- a/core/src/main/java/com/gitee/kooder/utils/JsonUtils.java +++ b/core/src/main/java/com/gitee/kooder/utils/JsonUtils.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.utils; import com.fasterxml.jackson.annotation.JsonInclude; diff --git a/core/src/main/java/com/gitee/kooder/utils/LanguageQuote.java b/core/src/main/java/com/gitee/kooder/utils/LanguageQuote.java index 13578ff3019640f4d419e75ae46df81bbd719373..bd33330e2b7d4f902ad8b940024239f7372367ff 100644 --- a/core/src/main/java/com/gitee/kooder/utils/LanguageQuote.java +++ b/core/src/main/java/com/gitee/kooder/utils/LanguageQuote.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.utils; /** diff --git a/core/src/main/java/com/gitee/kooder/utils/SlocCounter.java b/core/src/main/java/com/gitee/kooder/utils/SlocCounter.java index 9d7f7ede407f078de9c126812b651f8ce398464d..62627d999c588cc8c545bdc010e60ee4e772a401 100644 --- a/core/src/main/java/com/gitee/kooder/utils/SlocCounter.java +++ b/core/src/main/java/com/gitee/kooder/utils/SlocCounter.java @@ -43,10 +43,6 @@ public class SlocCounter { )); } - public ArrayList> getByteOrderMarks() { - return byteOrderMarks; - } - public boolean checkForMatch(char currentByte, int index, int endPoint, String[] matches, String content) { if (matches == null) { return false; @@ -338,11 +334,11 @@ public class SlocCounter { * found inside the file it was asked to count. */ public class SlocCount { - public int linesCount = 0; - public int blankCount = 0; - public int codeCount = 0; - public int commentCount = 0; - public int complexity = 0; + public int linesCount = 0; // file lines + public int blankCount = 0; // blank code lines + public int codeCount = 0; // code lines + public int commentCount = 0; // comment lines count + public int complexity = 0; // code complexity public SlocCount() { } diff --git a/core/src/main/java/com/gitee/kooder/utils/TextFileUtils.java b/core/src/main/java/com/gitee/kooder/utils/TextFileUtils.java index 2527b5d4e24cc8ae3ad2b005a15fffe65958e01f..136d905cc5380c4c6dbae872733c0a87c8e71aac 100644 --- a/core/src/main/java/com/gitee/kooder/utils/TextFileUtils.java +++ b/core/src/main/java/com/gitee/kooder/utils/TextFileUtils.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.utils; import com.glaforge.i18n.io.CharsetToolkit; @@ -44,17 +59,6 @@ public class TextFileUtils { return true; } - /* - if (!this.isNullEmptyOrWhitespace((String) this.properties.get(Values.DIRECTORY_BLACK_LIST))) { - String[] toIgnoreArray = ((String) this.properties.get(Values.DIRECTORY_BLACK_LIST)).split(","); - - for (String toIgnore : toIgnoreArray) { - if (fileParent.endsWith("/" + toIgnore) || fileParent.endsWith("/" + toIgnore + "/")) { - return true; - } - } - }*/ - return false; } @@ -65,29 +69,9 @@ public class TextFileUtils { * @param maxFileLineDepth 只返回最开始的 N 行,如果该值为负数,则返回所有行 */ public static List readFileLines(File file, int maxFileLineDepth) throws IOException { - return readFileLines(new FileInputStream(file), maxFileLineDepth); - /* - Charset charset = guessCharset(file); - StringBuilder stringBuilder = new StringBuilder(); - - try (BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), charset))){ - int i = 0; - int count = 0; - char[] buffer = new char[128]; - do { - int rc = reader.read(buffer); - if(rc > 0) { - stringBuilder.append(buffer, 0, rc); - count += rc; - } - if(rc == -1 || count >= DEFAULT_MAX_FILE_LENGTH_READ) - break; - } while (true); + try (InputStream stram = new FileInputStream(file)) { + return readFileLines(stram, maxFileLineDepth); } - - String[] lines = stringBuilder.toString().split("\\r\\n|\\n|\\r"); - return (lines.length > maxFileLineDepth && maxFileLineDepth > 0)?Arrays.asList(lines).subList(0, maxFileLineDepth):Arrays.asList(lines); - */ } /** diff --git a/core/src/main/resources/kooder.properties b/core/src/main/resources/kooder.properties index 86ade723711f1cc88deb67f9d2484bcee1fbca95..b38e28eaae81ef3c98357d836cd6e42d1b40aed3 100644 --- a/core/src/main/resources/kooder.properties +++ b/core/src/main/resources/kooder.properties @@ -8,9 +8,8 @@ http.log.pattern = /,/index/*,/search/*,/api/* http.webroot = gateway/src/main/webapp http.startup.tasks = indexer,gitlab -#gitlab.url = http://192.168.1.25:10080/ -gitlab.url = http://154.85.53.95:10080/ -gitlab.personal_access_token = Bt1H3ZUkD2bBFMbxxPxw +gitlab.url = +gitlab.personal_access_token = gitlab.secret_token = gsearch # Git ˺ãusername ssh ͬʱã @@ -29,8 +28,6 @@ queue.redis.key = gsearch-queue # queue.embed.url = http://127.0.0.1:8080/queue/fetch queue.embed.path = ./data/queue queue.embed.batch_size = 10000 -queue.test.auto_generate_task = true -queue.test.types = repo,issue,code # 洢 storage.type = disk diff --git a/docs/API.md b/docs/API.md new file mode 100644 index 0000000000000000000000000000000000000000..f0a2c9df28714ef189c109e89f4227de53241bc0 --- /dev/null +++ b/docs/API.md @@ -0,0 +1,40 @@ +### Kooder Search API + +**搜索接口** + +/search/repositories #仓库搜索 + +|参数名 |参数含义 | 示例| +--- | --- | --- +|q|搜索关键字|q=password| +|lang|指定编程语言(不支持多值)|lang=Java| +|e.id|搜索指定企业的仓库(仅限 gitee)|e.id=1213| +|sort|排序方法(stars,forks,update)|sort=update| +|p|页码(每页20条)|p=3| + +/search/codes #代码搜索 + +|参数名 |参数含义 | 示例| +--- | --- | --- +|q|搜索关键字|q=password| +|lang|指定编程语言(不支持多值)|lang=Java| +|e.id|搜索指定企业的仓库(仅限 gitee)|e.id=1213| +|repo.id|搜索指定仓库的代码,支持多值,使用逗号隔开|repo.id=1213,32| +|sort|排序方法(stars,forks,update)|sort=update| +|p|页码(每页20条)|p=3| + +/search/issues #Issue 搜索 + +|参数名 |参数含义 | 示例| +--- | --- | --- +|q|搜索关键字|q=password| +|e.id|搜索指定企业的仓库(仅限 gitee)|e.id=1213| +|sort|排序方法(create,update)|sort=update| +|p|页码(每页20条)|p=3| + + +**WebHook 回调接口** + +/gitlab/system # Gitlab 系统回调接口 +/gitlab/project # Gitlab 仓库回调接口 +/gitee # Gitee Premium 回调接口 diff --git a/configuration.md b/docs/configuration.md similarity index 100% rename from configuration.md rename to docs/configuration.md diff --git a/gateway/src/main/java/com/gitee/kooder/SearchCmd.java b/gateway/src/main/java/com/gitee/kooder/SearchCmd.java index f044c6be695174ee33c355dc1359b49297f4ee09..ace8eb213ac518b62f9f956a1f55193df6184632 100644 --- a/gateway/src/main/java/com/gitee/kooder/SearchCmd.java +++ b/gateway/src/main/java/com/gitee/kooder/SearchCmd.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder; import com.gitee.kooder.core.Constants; diff --git a/gateway/src/main/java/com/gitee/kooder/action/GiteaAction.java b/gateway/src/main/java/com/gitee/kooder/action/GiteaAction.java index 97a180ca4a9188eafd8181a1dc4d5cc28f95b8b1..03cd304c4d5a7492a7ecd81606120ec73e40628e 100644 --- a/gateway/src/main/java/com/gitee/kooder/action/GiteaAction.java +++ b/gateway/src/main/java/com/gitee/kooder/action/GiteaAction.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.action; import com.gitee.kooder.server.Action; diff --git a/gateway/src/main/java/com/gitee/kooder/action/GiteeAction.java b/gateway/src/main/java/com/gitee/kooder/action/GiteeAction.java index c8c3d65cdb648c97bfca67bbdeadeee1aea50cf6..ae66556af0b4d9a3aa1215067f587d4db67aca74 100644 --- a/gateway/src/main/java/com/gitee/kooder/action/GiteeAction.java +++ b/gateway/src/main/java/com/gitee/kooder/action/GiteeAction.java @@ -1,10 +1,24 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.action; import com.gitee.kooder.core.Constants; import com.gitee.kooder.core.GiteeSearchConfig; -import com.gitee.kooder.exception.GiteeException; +import com.gitee.kooder.gitee.GiteeException; import com.gitee.kooder.server.Action; -import com.gitee.kooder.server.GiteeWebHookManager; import io.vertx.ext.web.RoutingContext; /** @@ -15,7 +29,7 @@ import io.vertx.ext.web.RoutingContext; */ public class GiteeAction implements Action { - private static final String SECRET_TOKEN = GiteeSearchConfig.getProperty("gitlab.secret_token", Constants.DEFAULT_SECRET_TOKEN); + String SECRET_TOKEN = GiteeSearchConfig.getProperty("gitee.secret_token", Constants.DEFAULT_SECRET_TOKEN); /** * Gitee webhook handler diff --git a/gateway/src/main/java/com/gitee/kooder/server/GiteeWebHookManager.java b/gateway/src/main/java/com/gitee/kooder/action/GiteeWebHookManager.java similarity index 33% rename from gateway/src/main/java/com/gitee/kooder/server/GiteeWebHookManager.java rename to gateway/src/main/java/com/gitee/kooder/action/GiteeWebHookManager.java index 6e3df18fa825d507f63b0e48199d7d24654dc20e..0da9e850b3a79cdbfd7b2f918e65213c23516f9a 100644 --- a/gateway/src/main/java/com/gitee/kooder/server/GiteeWebHookManager.java +++ b/gateway/src/main/java/com/gitee/kooder/action/GiteeWebHookManager.java @@ -1,83 +1,133 @@ -package com.gitee.kooder.server; +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.gitee.kooder.action; import com.gitee.kooder.core.Constants; -import com.gitee.kooder.exception.GiteeException; +import com.gitee.kooder.gitee.*; import com.gitee.kooder.models.CodeRepository; -import com.gitee.kooder.models.GiteeWebHookEvent; import com.gitee.kooder.models.Issue; +import com.gitee.kooder.models.Relation; import com.gitee.kooder.models.Repository; -import com.gitee.kooder.models.gitee.IssueWebHook; -import com.gitee.kooder.models.gitee.RepoWebHook; import com.gitee.kooder.queue.QueueTask; import com.gitee.kooder.utils.JsonUtils; import io.vertx.ext.web.RoutingContext; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.Objects; -import java.util.Optional; /** + * This class provides a handler for processing gitee Web Hook callouts. + * * @author zhanggx */ -public class GiteeWebHookManager { - - private final static Logger log = LoggerFactory.getLogger(GiteeWebHookManager.class); +class GiteeWebHookManager { + /** + * handle gitee web hook + * + * @param secretToken + * @param context + * @throws GiteeException + */ public static void handleEvent(String secretToken, RoutingContext context) throws GiteeException { checkSecretToken(secretToken, context); GiteeWebHookEvent giteeWebHookEvent = parseGiteeWebHookEvent(context); + Enterprise enterprise = GiteeApi.getInstance().getEnterprise(); switch (giteeWebHookEvent) { - case REPO_HOO: - RepoWebHook repoWebHook = JsonUtils.readValue(context.getBodyAsString(), RepoWebHook.class); - if (repoWebHook == null) { - throw new GiteeException("Cannot resolve repo web hook!"); - } - - if (RepoWebHook.ACTION_CREATE.equals(repoWebHook.getAction())) { - Repository repository = new Repository(repoWebHook.getRepository()); - QueueTask.add(Constants.TYPE_REPOSITORY, repository); - - CodeRepository coder = new CodeRepository(repository); - coder.setScm(CodeRepository.SCM_GIT); - QueueTask.add(Constants.TYPE_CODE, coder); - - } else if (RepoWebHook.ACTION_DESTROY.equals(repoWebHook.getAction())) { - CodeRepository codeRepository = new CodeRepository(); - codeRepository.setId(repoWebHook.getRepository().getId()); - QueueTask.delete(Constants.TYPE_CODE, codeRepository); - - Repository repository = new Repository(); - repository.setId(repoWebHook.getRepository().getId()); - QueueTask.delete(Constants.TYPE_REPOSITORY, repository); - - } else { - throw new GiteeException("Repo web hook action unsupported!"); - } - + case REPO_HOOK: + handleRepoHookEvent(context, enterprise); break; case PUSH_HOOK: - Integer projectId = Optional.ofNullable(context.getBodyAsJson()) - .map(jsonObject -> jsonObject.getJsonObject("project")) - .map(jsonObject -> jsonObject.getInteger("id")) - .orElseThrow(() -> new GiteeException("Cannot resolve project id!")); - QueueTask.update(Constants.TYPE_CODE, new CodeRepository(projectId)); + handlePushHookEvent(context, enterprise); break; case ISSUE_HOOK: - IssueWebHook issueWebHook = JsonUtils.readValue(context.getBodyAsString(), IssueWebHook.class); - if (issueWebHook == null) { - throw new GiteeException("Cannot resolve issue web hook!"); - } - issueWebHook.getIssue().setRepository(issueWebHook.getRepository()); - Issue issue = new Issue(issueWebHook.getIssue()); - QueueTask.update(Constants.TYPE_ISSUE, issue); + handleIssueHookEvent(context, enterprise); break; default: throw new GiteeException("Web hook event unsupported!"); } } + /** + * handle repo hook + * + * @param context + * @param enterprise + * @throws GiteeException action unsupported! + */ + private static void handleRepoHookEvent(RoutingContext context, Enterprise enterprise) throws GiteeException { + RepoWebHook repoWebHook = JsonUtils.readValue(context.getBodyAsString(), RepoWebHook.class); + if (repoWebHook == null) { + throw new GiteeException("Cannot resolve repo web hook!"); + } + + if (RepoWebHook.ACTION_CREATE.equals(repoWebHook.getAction())) { + // index repository data + Repository repository = repoWebHook.getRepository().toKooderRepository(); + repository.setEnterprise(new Relation(enterprise.getId(), enterprise.getName(), enterprise.getUrl())); + QueueTask.add(Constants.TYPE_REPOSITORY, repository); + + } else if (RepoWebHook.ACTION_DESTROY.equals(repoWebHook.getAction())) { + // delete repository data + Repository repository = new Repository(); + repository.setId(repoWebHook.getRepository().getId()); + QueueTask.delete(Constants.TYPE_REPOSITORY, repository); + + } else { + throw new GiteeException("Repo web hook action unsupported!"); + } + } + + /** + * handle push hook + * + * @param context + * @param enterprise + * @throws GiteeException cannot resolve push web hook + */ + private static void handlePushHookEvent(RoutingContext context, Enterprise enterprise) throws GiteeException { + PushWebHook pushWebHook = JsonUtils.readValue(context.getBodyAsString(), PushWebHook.class); + if (pushWebHook == null) { + throw new GiteeException("Cannot resolve push web hook!"); + } + Repository repository = pushWebHook.getRepository().toKooderRepository(); + repository.setEnterprise(new Relation(enterprise.getId(), enterprise.getName(), enterprise.getUrl())); + CodeRepository codeRepository = new CodeRepository(repository); + codeRepository.setScm(CodeRepository.SCM_GIT); + QueueTask.add(Constants.TYPE_CODE, codeRepository); + } + + /** + * handle issue hook + * + * @param context + * @param enterprise + * @throws GiteeException cannot resolve issue web hook + */ + private static void handleIssueHookEvent(RoutingContext context, Enterprise enterprise) throws GiteeException { + IssueWebHook issueWebHook = JsonUtils.readValue(context.getBodyAsString(), IssueWebHook.class); + if (issueWebHook == null) { + throw new GiteeException("Cannot resolve issue web hook!"); + } + issueWebHook.getIssue().setRepository(issueWebHook.getRepository()); + Issue issue = issueWebHook.getIssue().toKooderIssue(); + issue.setEnterprise(new Relation(enterprise.getId(), enterprise.getName(), enterprise.getUrl())); + QueueTask.update(Constants.TYPE_ISSUE, issue); + } + /** * check web hook secret token * @@ -104,7 +154,7 @@ public class GiteeWebHookManager { private static GiteeWebHookEvent parseGiteeWebHookEvent(RoutingContext context) throws GiteeException { String event = context.request().getHeader("X-Gitee-Event"); if (event == null) { - throw new GiteeException("X-Gitlab-Event header is missing!"); + throw new GiteeException("X-Gitee-Event header is missing!"); } else { GiteeWebHookEvent giteeWebHookEvent = GiteeWebHookEvent.getEvent(event); if (Objects.nonNull(giteeWebHookEvent)) { diff --git a/gateway/src/main/java/com/gitee/kooder/action/GitlabAction.java b/gateway/src/main/java/com/gitee/kooder/action/GitlabAction.java index 50b80456ece8832b9805bd42a59f39fe3788854c..3277df9200498469b247049ccfd9a5088b6f25d6 100644 --- a/gateway/src/main/java/com/gitee/kooder/action/GitlabAction.java +++ b/gateway/src/main/java/com/gitee/kooder/action/GitlabAction.java @@ -1,16 +1,28 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.action; import com.gitee.kooder.models.CodeRepository; import com.gitee.kooder.core.Constants; import com.gitee.kooder.core.GiteeSearchConfig; import com.gitee.kooder.indexer.Gitlab; -import com.gitee.kooder.models.GitlabProjectEvent; import com.gitee.kooder.models.Issue; import com.gitee.kooder.models.Repository; import com.gitee.kooder.queue.QueueTask; import com.gitee.kooder.server.Action; -import com.gitee.kooder.server.GitlabSystemHookManager; -import com.gitee.kooder.server.GitlabWebhookManager; import io.vertx.ext.web.RoutingContext; import org.gitlab4j.api.GitLabApiException; import org.gitlab4j.api.models.Project; @@ -48,18 +60,9 @@ public class GitlabAction implements Action { if(p != null) { Repository repo = new Repository(p); QueueTask.add(Constants.TYPE_REPOSITORY, repo); - - CodeRepository coder = new CodeRepository(repo); - coder.setScm(CodeRepository.SCM_GIT); - QueueTask.add(Constants.TYPE_CODE, coder); } break; case GitlabProjectEvent.E_PROJECT_DESTROY: - //delete all source codes of this repository - CodeRepository coder = new CodeRepository(); - coder.setId(event.getProjectId()); - QueueTask.delete(Constants.TYPE_CODE, coder); - //delete repository Repository repo = new Repository(); repo.setId(event.getProjectId()); QueueTask.delete(Constants.TYPE_REPOSITORY, repo); @@ -78,9 +81,14 @@ public class GitlabAction implements Action { this.fireCodeUpdate(event.getProjectId()); } - private void fireCodeUpdate(long pid) { - CodeRepository repo = new CodeRepository(pid); - QueueTask.update(Constants.TYPE_CODE, repo); //update source code indexes + private void fireCodeUpdate(int pid) { + Project p = getProject(pid); + if(p != null) { + Repository repo = new Repository(p); + CodeRepository coder = new CodeRepository(repo); + coder.setScm(CodeRepository.SCM_GIT); + QueueTask.add(Constants.TYPE_CODE, coder); //update source code indexes + } } }; hookMgr.handleEvent(context); @@ -116,5 +124,18 @@ public class GitlabAction implements Action { return null; } + /** + * Gitlab repositories events definition + */ + class GitlabProjectEvent { + + public final static String E_PROJECT_CREATE = "project_create"; + public final static String E_PROJECT_DESTROY = "project_destroy"; + public final static String E_PROJECT_RENAME = "project_rename"; + public final static String E_PROJECT_TRANSFER = "project_transfer"; + public final static String E_PROJECT_UPDATE = "project_update"; + + } + } diff --git a/gateway/src/main/java/com/gitee/kooder/server/GitlabSystemHookManager.java b/gateway/src/main/java/com/gitee/kooder/action/GitlabSystemHookManager.java similarity index 92% rename from gateway/src/main/java/com/gitee/kooder/server/GitlabSystemHookManager.java rename to gateway/src/main/java/com/gitee/kooder/action/GitlabSystemHookManager.java index a185b535cff91913737fee734527a49700765124..1e08aaa835f028b7125991305009cc9122d7afcf 100644 --- a/gateway/src/main/java/com/gitee/kooder/server/GitlabSystemHookManager.java +++ b/gateway/src/main/java/com/gitee/kooder/action/GitlabSystemHookManager.java @@ -1,4 +1,19 @@ -package com.gitee.kooder.server; +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.gitee.kooder.action; import io.vertx.ext.web.RoutingContext; import org.apache.commons.lang3.StringUtils; @@ -16,7 +31,7 @@ import org.slf4j.LoggerFactory; * This class provides a handler for processing GitLab System Hook callouts. * @author Winter Lau */ -public class GitlabSystemHookManager { +class GitlabSystemHookManager { private final static Logger log = LoggerFactory.getLogger(GitlabSystemHookManager.class); diff --git a/gateway/src/main/java/com/gitee/kooder/server/GitlabWebhookManager.java b/gateway/src/main/java/com/gitee/kooder/action/GitlabWebhookManager.java similarity index 91% rename from gateway/src/main/java/com/gitee/kooder/server/GitlabWebhookManager.java rename to gateway/src/main/java/com/gitee/kooder/action/GitlabWebhookManager.java index 502fd2f12754a42d428137ddf2e3a968b0f385f2..d0d1f7401e3b914a302826997334a7715c405b3e 100644 --- a/gateway/src/main/java/com/gitee/kooder/server/GitlabWebhookManager.java +++ b/gateway/src/main/java/com/gitee/kooder/action/GitlabWebhookManager.java @@ -1,4 +1,19 @@ -package com.gitee.kooder.server; +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.gitee.kooder.action; import io.vertx.ext.web.RoutingContext; import org.apache.commons.lang3.StringUtils; @@ -13,7 +28,7 @@ import org.slf4j.LoggerFactory; * This class provides a handler for processing GitLab Web Hook callouts. * @author Winter Lau */ -public class GitlabWebhookManager { +class GitlabWebhookManager { private final static org.slf4j.Logger log = LoggerFactory.getLogger(GitlabWebhookManager.class); diff --git a/gateway/src/main/java/com/gitee/kooder/action/IndexAction.java b/gateway/src/main/java/com/gitee/kooder/action/IndexAction.java index a841ef5523085fb08e38117128776e3480f342fa..774ddaaac1d020e11051d4fcd49141f231840d8d 100644 --- a/gateway/src/main/java/com/gitee/kooder/action/IndexAction.java +++ b/gateway/src/main/java/com/gitee/kooder/action/IndexAction.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.action; import com.gitee.kooder.server.Action; diff --git a/gateway/src/main/java/com/gitee/kooder/action/SearchAction.java b/gateway/src/main/java/com/gitee/kooder/action/SearchAction.java index 09ebbdb089e67398698576c6dee89111f58233a8..57632cb84f17631e47ed3bc60b2ba9bb475adde0 100644 --- a/gateway/src/main/java/com/gitee/kooder/action/SearchAction.java +++ b/gateway/src/main/java/com/gitee/kooder/action/SearchAction.java @@ -1,7 +1,25 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.action; import com.gitee.kooder.core.Constants; +import com.gitee.kooder.core.SearchHelper; import com.gitee.kooder.models.QueryResult; +import com.gitee.kooder.models.Searchable; +import com.gitee.kooder.models.SourceFile; import com.gitee.kooder.query.QueryFactory; import io.netty.handler.codec.http.HttpResponseStatus; import io.vertx.core.http.HttpServerRequest; @@ -19,6 +37,8 @@ import java.util.stream.Collectors; */ public class SearchAction implements SearchActionBase { + private final static int MAX_LINES = 20; // max code lines in highlight + /** * controller for search.vm * @param context @@ -63,7 +83,7 @@ public class SearchAction implements SearchActionBase { String body = context.getBodyAsString(); if(body != null) repos.addAll(Arrays.asList(body.split(","))); - List iRepos = repos.stream().map(r -> NumberUtils.toInt(r, 0)).filter(r -> (r > 0)).collect(Collectors.toList()); + List iRepos = repos.stream().map(r -> NumberUtils.toLong(r, 0)).filter(r -> (r > 0)).collect(Collectors.toList()); QueryResult result = QueryFactory.REPO() .setEnterpriseId(eid) @@ -114,7 +134,7 @@ public class SearchAction implements SearchActionBase { String body = context.getBodyAsString(); if(body != null) repos.addAll(Arrays.asList(body.split(","))); - List iRepos = repos.stream().map(r -> NumberUtils.toInt(r, 0)).filter(r -> (r > 0)).collect(Collectors.toList()); + List iRepos = repos.stream().map(r -> NumberUtils.toLong(r, 0)).filter(r -> (r > 0)).collect(Collectors.toList()); QueryResult result = QueryFactory.CODE() .setEnterpriseId(eid) @@ -128,6 +148,11 @@ public class SearchAction implements SearchActionBase { .setPageSize(PAGE_SIZE) .execute(); + for(Searchable obj : result.getObjects()) { + SourceFile file = (SourceFile) obj; + file.setResult(SearchHelper.hl_lines(file.getContents(), q, MAX_LINES)); + } + this.json(context.response(), result.json()); } diff --git a/gateway/src/main/java/com/gitee/kooder/action/SearchActionBase.java b/gateway/src/main/java/com/gitee/kooder/action/SearchActionBase.java index 5bf5e366d1e07aeed849deaebc745c550e9698c7..d098c8e55770d3fbce6a95f2a021e47c662fd3b5 100644 --- a/gateway/src/main/java/com/gitee/kooder/action/SearchActionBase.java +++ b/gateway/src/main/java/com/gitee/kooder/action/SearchActionBase.java @@ -1,10 +1,24 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.action; import com.gitee.kooder.core.Constants; import com.gitee.kooder.models.QueryResult; import com.gitee.kooder.query.QueryFactory; import com.gitee.kooder.server.Action; -import io.vertx.core.http.HttpServerRequest; import io.vertx.ext.web.RoutingContext; import org.apache.commons.lang3.StringUtils; @@ -15,7 +29,7 @@ import java.util.Arrays; * Action for search, both for web and api * @author Winter Lau */ -public interface SearchActionBase extends Action { +interface SearchActionBase extends Action { /** * execute search diff --git a/gateway/src/main/java/com/gitee/kooder/action/TaskAction.java b/gateway/src/main/java/com/gitee/kooder/action/TaskAction.java index 2f34499cdcdad27d5fc37a112b288c840c42c624..3a0a1aed43dc01e7ac540e41eaf12ff8604b1e22 100644 --- a/gateway/src/main/java/com/gitee/kooder/action/TaskAction.java +++ b/gateway/src/main/java/com/gitee/kooder/action/TaskAction.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.action; import com.gitee.kooder.core.Constants; diff --git a/gateway/src/main/java/com/gitee/kooder/examples/RangeFacetsExample.java b/gateway/src/main/java/com/gitee/kooder/examples/RangeFacetsExample.java index 894a01febe19e38a30b76ff9f0abed0e47aed551..1b83e63c51aa04e261bbe661a4e24fa503229290 100644 --- a/gateway/src/main/java/com/gitee/kooder/examples/RangeFacetsExample.java +++ b/gateway/src/main/java/com/gitee/kooder/examples/RangeFacetsExample.java @@ -126,26 +126,25 @@ public class RangeFacetsExample implements Closeable { /** Runs the search and drill-down examples and prints the results. */ public static void main(String[] args) throws Exception { - RangeFacetsExample example = new RangeFacetsExample(); - example.index(); - - System.out.println("Facet counting example:"); - System.out.println("-----------------------"); - System.out.println(example.search()); - - System.out.println("\n"); - System.out.println("Facet drill-down example (timestamp/Past six hours):"); - System.out.println("---------------------------------------------"); - TopDocs hits = example.drillDown(example.PAST_SIX_HOURS); - System.out.println(hits.totalHits + " totalHits"); - - System.out.println("\n"); - System.out.println("Facet drill-sideways example (timestamp/Past six hours):"); - System.out.println("---------------------------------------------"); - DrillSideways.DrillSidewaysResult sideways = example.drillSideways(example.PAST_SIX_HOURS); - System.out.println(sideways.hits.totalHits + " totalHits"); - System.out.println(sideways.facets.getTopChildren(10, "timestamp")); - - example.close(); + try(RangeFacetsExample example = new RangeFacetsExample()) { + example.index(); + + System.out.println("Facet counting example:"); + System.out.println("-----------------------"); + System.out.println(example.search()); + + System.out.println("\n"); + System.out.println("Facet drill-down example (timestamp/Past six hours):"); + System.out.println("---------------------------------------------"); + TopDocs hits = example.drillDown(example.PAST_SIX_HOURS); + System.out.println(hits.totalHits + " totalHits"); + + System.out.println("\n"); + System.out.println("Facet drill-sideways example (timestamp/Past six hours):"); + System.out.println("---------------------------------------------"); + DrillSideways.DrillSidewaysResult sideways = example.drillSideways(example.PAST_SIX_HOURS); + System.out.println(sideways.hits.totalHits + " totalHits"); + System.out.println(sideways.facets.getTopChildren(10, "timestamp")); + } } } \ No newline at end of file diff --git a/gateway/src/main/java/com/gitee/kooder/server/Action.java b/gateway/src/main/java/com/gitee/kooder/server/Action.java index e6583f3e39512c70fff40e8ad0e69657d7499772..c8b58cf2bd535142d1beccc31d278d36ea7bc020 100644 --- a/gateway/src/main/java/com/gitee/kooder/server/Action.java +++ b/gateway/src/main/java/com/gitee/kooder/server/Action.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.server; import com.gitee.kooder.queue.QueueTask; @@ -15,7 +30,7 @@ import java.util.Map; /** * Action base - * Eash request has one independent action instance + * Each request has one independent action instance * @author Winter Lau */ public interface Action { diff --git a/gateway/src/main/java/com/gitee/kooder/server/ActionExecutor.java b/gateway/src/main/java/com/gitee/kooder/server/ActionExecutor.java index 7ea47f7fc17bc632931abbf432cd845d0682cb46..8334c77a675dea38d10b3156dfff23491bba73d0 100644 --- a/gateway/src/main/java/com/gitee/kooder/server/ActionExecutor.java +++ b/gateway/src/main/java/com/gitee/kooder/server/ActionExecutor.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.server; import io.netty.handler.codec.http.HttpResponseStatus; diff --git a/gateway/src/main/java/com/gitee/kooder/server/AutoContentTypeStaticHandler.java b/gateway/src/main/java/com/gitee/kooder/server/AutoContentTypeStaticHandler.java index c56020e228721a3e8ab1e11c049df5f251ad2967..1292c64d261b5ca71b30d989519a89c04864761d 100644 --- a/gateway/src/main/java/com/gitee/kooder/server/AutoContentTypeStaticHandler.java +++ b/gateway/src/main/java/com/gitee/kooder/server/AutoContentTypeStaticHandler.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.server; import com.gitee.kooder.core.GiteeSearchConfig; diff --git a/gateway/src/main/java/com/gitee/kooder/server/Gateway.java b/gateway/src/main/java/com/gitee/kooder/server/Gateway.java index ec1a42b0fa149fd4a2a432e27d2d07d5b7041711..786187bf0ebfda01f588935c0636adba7a5ddf24 100644 --- a/gateway/src/main/java/com/gitee/kooder/server/Gateway.java +++ b/gateway/src/main/java/com/gitee/kooder/server/Gateway.java @@ -1,9 +1,24 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.server; import com.gitee.kooder.core.GiteeSearchConfig; import com.gitee.kooder.indexer.FetchTaskThread; import com.gitee.kooder.indexer.GiteaIndexThread; -import com.gitee.kooder.indexer.GiteeIndexThread; +import com.gitee.kooder.gitee.GiteeIndexThread; import com.gitee.kooder.indexer.GitlabIndexThread; import io.vertx.core.http.HttpServerResponse; import io.vertx.core.net.SocketAddress; @@ -21,7 +36,7 @@ import java.util.Map; public class Gateway extends GatewayBase { private final static String pattern_static_file = "/.*\\.(css|ico|js|html|htm|jpg|png|gif)"; - private final static Map startupTasks = new HashMap<>(){{ + private final static Map startupTasks = new HashMap(){{ put("indexer", new FetchTaskThread()); put("gitlab", new GitlabIndexThread()); put("gitee", new GiteeIndexThread()); diff --git a/gateway/src/main/java/com/gitee/kooder/server/GatewayBase.java b/gateway/src/main/java/com/gitee/kooder/server/GatewayBase.java index 88116328fe09832d1c25cdaf562d5186b93a9d37..e31e1afa4dfd70d97d2a1686a0c132fbfa1ca0dd 100644 --- a/gateway/src/main/java/com/gitee/kooder/server/GatewayBase.java +++ b/gateway/src/main/java/com/gitee/kooder/server/GatewayBase.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.server; import com.gitee.kooder.core.GiteeSearchConfig; diff --git a/gateway/src/main/java/com/gitee/kooder/server/TemplateEngine.java b/gateway/src/main/java/com/gitee/kooder/server/TemplateEngine.java index fd4c065a772fca4455a8647bde898b2193094bbd..36be709aa328eff1cf3c84afb25800ae4e2e2c4b 100644 --- a/gateway/src/main/java/com/gitee/kooder/server/TemplateEngine.java +++ b/gateway/src/main/java/com/gitee/kooder/server/TemplateEngine.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.server; import com.gitee.kooder.core.GiteeSearchConfig; diff --git a/gateway/src/main/java/com/gitee/kooder/server/Tester.java b/gateway/src/main/java/com/gitee/kooder/server/Tester.java index a87a084e50938cd643dc26158190323cd9dfbfd4..f7bb99c2fddbca12b118614630b649dbdd20dcde 100644 --- a/gateway/src/main/java/com/gitee/kooder/server/Tester.java +++ b/gateway/src/main/java/com/gitee/kooder/server/Tester.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.server; import org.apache.commons.lang3.StringUtils; diff --git a/gateway/src/main/java/com/gitee/kooder/server/VelocityTool.java b/gateway/src/main/java/com/gitee/kooder/server/VelocityTool.java index d67b51eea3bb689ce23ec107a43862ffc5ad5ee2..256c548b76e9dc9ab56cae4235950ba19d8f6280 100644 --- a/gateway/src/main/java/com/gitee/kooder/server/VelocityTool.java +++ b/gateway/src/main/java/com/gitee/kooder/server/VelocityTool.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.server; import com.gitee.kooder.models.CodeLine; diff --git a/indexer/src/main/java/com/gitee/kooder/gitee/Enterprise.java b/indexer/src/main/java/com/gitee/kooder/gitee/Enterprise.java new file mode 100644 index 0000000000000000000000000000000000000000..48f5469e98a1acba396fe801b26e90f8301ebe3b --- /dev/null +++ b/indexer/src/main/java/com/gitee/kooder/gitee/Enterprise.java @@ -0,0 +1,51 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.gitee.kooder.gitee; + +/** + * @author zhanggx + */ +public class Enterprise { + + private Integer id; + private String name; + private String url; + + public Integer getId() { + return id; + } + + public void setId(Integer id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + +} diff --git a/indexer/src/main/java/com/gitee/kooder/gitee/EnterpriseHook.java b/indexer/src/main/java/com/gitee/kooder/gitee/EnterpriseHook.java new file mode 100644 index 0000000000000000000000000000000000000000..191439fa0db8edcdf00bfb228d3ad4cc6c6530fa --- /dev/null +++ b/indexer/src/main/java/com/gitee/kooder/gitee/EnterpriseHook.java @@ -0,0 +1,33 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.gitee.kooder.gitee; + +/** + * @author zhanggx + */ +public class EnterpriseHook { + + private String url; + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + +} diff --git a/core/src/main/java/com/gitee/kooder/api/GiteeApi.java b/indexer/src/main/java/com/gitee/kooder/gitee/GiteeApi.java similarity index 70% rename from core/src/main/java/com/gitee/kooder/api/GiteeApi.java rename to indexer/src/main/java/com/gitee/kooder/gitee/GiteeApi.java index defe90913821e61469d6b7bf681ac4c791f86421..4448342ba9b6f464b610bd74ac6ba3624f3a2ef4 100644 --- a/core/src/main/java/com/gitee/kooder/api/GiteeApi.java +++ b/indexer/src/main/java/com/gitee/kooder/gitee/GiteeApi.java @@ -1,11 +1,22 @@ -package com.gitee.kooder.api; +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.gitee.kooder.gitee; import com.fasterxml.jackson.core.type.TypeReference; import com.gitee.kooder.core.GiteeSearchConfig; -import com.gitee.kooder.exception.GiteeException; -import com.gitee.kooder.models.gitee.EnterpriseHook; -import com.gitee.kooder.models.gitee.Issue; -import com.gitee.kooder.models.gitee.Repository; import com.gitee.kooder.utils.HttpUtils; import com.gitee.kooder.utils.JsonUtils; import okhttp3.Response; @@ -15,29 +26,69 @@ import org.slf4j.LoggerFactory; import java.util.*; /** + * gitee open api + * * @author zhanggx */ public class GiteeApi { private final static Logger log = LoggerFactory.getLogger(GiteeApi.class); + private static final String URL_GET_ENTERPRISE = "/api/v5/enterprises/enterprise"; private static final String URL_CREATE_ENTERPRISE_HOOKS = "/api/v5/enterprises/enterprise/hooks/create"; private static final String URL_GET_ENTERPRISE_HOOKS = "/api/v5/enterprises/enterprise/hooks"; private static final String URL_GET_REPOS = "/api/v5/enterprises/enterprise/repos"; private static final String URL_GET_ISSUES = "/api/v5/enterprises/enterprise/issues"; + /** + * gitee server url + */ public final String giteeUrl; - private final String presonalAccessToken; + /** + * personal access token + */ + private final String personalAccessToken; private GiteeApi() { giteeUrl = GiteeSearchConfig.getProperty("gitee.url"); - presonalAccessToken = GiteeSearchConfig.getProperty("gitee.personal_access_token"); + personalAccessToken = GiteeSearchConfig.getProperty("gitee.personal_access_token"); } public static GiteeApi getInstance() { return Singleton.INSTANCE; } + /** + * get enterprise info + * + * @return + */ + public Enterprise getEnterprise() throws GiteeException { + Map requestParamsMap = getRequestParamsMap(); + try (Response response = HttpUtils.get(giteeUrl + URL_GET_ENTERPRISE, requestParamsMap)) { + if (response.isSuccessful()) { + return JsonUtils.readValue(response.body().string(), Enterprise.class); + } + throw new GiteeException(response.body().string()); + } catch (Exception e) { + log.warn("Create gitee enterprise hooks error: {}", e.getMessage()); + throw new GiteeException(e.getMessage()); + } + } + + /** + * create enterprise hook + * + * @param url enterprise hook url + * @param secretToken enterprise hook password + * @param pushEvents notice push event + * @param repoEvents notice repo event + * @param tagPushEvents notice tag push event + * @param issuesEvents notice issue event + * @param noteEvents notice note event + * @param mergeRequestsEvents notice merge request event + * @throws GiteeException request gitee server error + */ public void createEnterpriseHooks(String url, String secretToken, boolean pushEvents, @@ -65,6 +116,12 @@ public class GiteeApi { } } + /** + * get all enterprise hook + * + * @return all enterprise hook + * @throws GiteeException + */ public List getEnterpriseHooks() throws GiteeException { int pageNo = 1, pageSize = 50; List res = new ArrayList<>(); @@ -75,6 +132,14 @@ public class GiteeApi { return res; } + /** + * paging get enterprise hook + * + * @param pageNo + * @param pageSize + * @return + * @throws GiteeException + */ private List getEnterpriseHooks(int pageNo, int pageSize) throws GiteeException { Map requestParamsMap = getRequestParamsMap(); requestParamsMap.put("page", String.valueOf(pageNo)); @@ -94,6 +159,13 @@ public class GiteeApi { } } + /** + * get all repo + * + * @param afterId + * @return + * @throws GiteeException + */ public List getRepos(int afterId) throws GiteeException { int pageNo = 1, pageSize = 50; List res = new ArrayList<>(); @@ -109,6 +181,14 @@ public class GiteeApi { return res; } + /** + * paging get repo + * + * @param pageNo + * @param pageSize + * @return + * @throws GiteeException + */ private List getRepos(int pageNo, int pageSize) throws GiteeException { Map requestParamsMap = getRequestParamsMap(); requestParamsMap.put("type", "all"); @@ -130,6 +210,13 @@ public class GiteeApi { } } + /** + * get all issue + * + * @param afterId + * @return + * @throws GiteeException + */ public List getIssues(int afterId) throws GiteeException { int pageNo = 1, pageSize = 50; List res = new ArrayList<>(); @@ -145,6 +232,14 @@ public class GiteeApi { return res; } + /** + * paging get issue + * + * @param pageNo + * @param pageSize + * @return + * @throws GiteeException + */ private List getIssues(int pageNo, int pageSize) throws GiteeException { Map requestParamsMap = getRequestParamsMap(); requestParamsMap.put("state", "all"); @@ -166,9 +261,14 @@ public class GiteeApi { } } + /** + * get request params map with personal access token + * + * @return + */ private Map getRequestParamsMap() { Map map = new HashMap<>(); - map.put("access_token", presonalAccessToken); + map.put("access_token", personalAccessToken); return map; } diff --git a/indexer/src/main/java/com/gitee/kooder/gitee/GiteeException.java b/indexer/src/main/java/com/gitee/kooder/gitee/GiteeException.java new file mode 100644 index 0000000000000000000000000000000000000000..31c344d34976a3fb057bff3d3a5a537444354602 --- /dev/null +++ b/indexer/src/main/java/com/gitee/kooder/gitee/GiteeException.java @@ -0,0 +1,29 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.gitee.kooder.gitee; + +/** + * Gitee related exception + * + * @author zhanggx + */ +public class GiteeException extends Exception { + + public GiteeException(String message) { + super(message); + } + +} diff --git a/indexer/src/main/java/com/gitee/kooder/indexer/GiteeIndexThread.java b/indexer/src/main/java/com/gitee/kooder/gitee/GiteeIndexThread.java similarity index 53% rename from indexer/src/main/java/com/gitee/kooder/indexer/GiteeIndexThread.java rename to indexer/src/main/java/com/gitee/kooder/gitee/GiteeIndexThread.java index 6394d54f519288b764baedcffd7fc68af92d6fb6..e59e7a01c92c0b160c650829a70396a2ae9b5331 100644 --- a/indexer/src/main/java/com/gitee/kooder/indexer/GiteeIndexThread.java +++ b/indexer/src/main/java/com/gitee/kooder/gitee/GiteeIndexThread.java @@ -1,13 +1,27 @@ -package com.gitee.kooder.indexer; +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.gitee.kooder.gitee; -import com.gitee.kooder.api.GiteeApi; import com.gitee.kooder.core.Constants; import com.gitee.kooder.core.GiteeSearchConfig; -import com.gitee.kooder.exception.GiteeException; +import com.gitee.kooder.indexer.GitlabIndexThread; import com.gitee.kooder.models.CodeRepository; import com.gitee.kooder.models.Issue; +import com.gitee.kooder.models.Relation; import com.gitee.kooder.models.Repository; -import com.gitee.kooder.models.gitee.EnterpriseHook; import com.gitee.kooder.query.QueryFactory; import com.gitee.kooder.queue.QueueTask; import org.slf4j.Logger; @@ -16,7 +30,7 @@ import org.slf4j.LoggerFactory; import java.util.List; /** - * TODO Check and Index all of Gitee data for first time + * Check and Index all of Gitee data for first time * * @author Winter Lau */ @@ -24,10 +38,17 @@ public class GiteeIndexThread extends Thread { private final static Logger log = LoggerFactory.getLogger("[gitee]"); - private final static int itemsPerPage = 20; - + /** + * local address + */ private String gsearchUrl; + /** + * web hook url + */ private String systemHookUrl; + /** + * web hook password + */ private String secretToken; public static void main(String[] args) { @@ -44,15 +65,21 @@ public class GiteeIndexThread extends Thread { public void run() { try { long ct = System.currentTimeMillis(); + Enterprise enterprise = GiteeApi.getInstance().getEnterprise(); checkAndInstallEnterpriseHook(); - checkAndIndexProjects(); - checkAndIndexIssues(); + checkAndIndexProjects(enterprise); + checkAndIndexIssues(enterprise); log.info("Gitee data initialize finished in {} ms.", System.currentTimeMillis() - ct); } catch (GiteeException e) { - log.error("Failed to initialize gitlab data.", e); + log.error("Failed to initialize gitee data.", e); } } + /** + * check and install enterprise hook + * + * @throws GiteeException install enterprise hook error + */ private void checkAndInstallEnterpriseHook() throws GiteeException { List enterpriseHookList = GiteeApi.getInstance().getEnterpriseHooks(); for (EnterpriseHook enterpriseHook : enterpriseHookList) { @@ -61,21 +88,29 @@ public class GiteeIndexThread extends Thread { } } GiteeApi.getInstance().createEnterpriseHooks(systemHookUrl, secretToken, true, true, false, true, false, false); - log.info("Gitlab system hook : {} installed.", systemHookUrl); + log.info("Gitee enterprise hook : {} installed.", systemHookUrl); } - private void checkAndIndexProjects() throws GiteeException { + /** + * check and index projects data + * + * @throws GiteeException get projects data error + * @param enterprise + */ + private void checkAndIndexProjects(Enterprise enterprise) throws GiteeException { long ct = System.currentTimeMillis(); int pc = 0; Repository lastRepository = (Repository) QueryFactory.REPO().getLastestObject(); int maxId = lastRepository == null ? 0 : Math.toIntExact(lastRepository.getId()); - List repositoryList = GiteeApi.getInstance().getRepos(maxId); - for (com.gitee.kooder.models.gitee.Repository repository : repositoryList) { - Repository repo = new Repository(repository); + List repositoryList = GiteeApi.getInstance().getRepos(maxId); + for (com.gitee.kooder.gitee.Repository repository : repositoryList) { + Repository repo = repository.toKooderRepository(); + repo.setEnterprise(new Relation(enterprise.getId(), enterprise.getName(), enterprise.getUrl())); QueueTask.add(Constants.TYPE_REPOSITORY, repo); CodeRepository codes = new CodeRepository(); codes.setId(repository.getId()); + codes.setEnterprise(enterprise.getId()); codes.setScm(CodeRepository.SCM_GIT); codes.setName(repository.getName()); codes.setUrl(repository.getHtmlUrl()); @@ -86,15 +121,23 @@ public class GiteeIndexThread extends Thread { log.info("{} repositories indexed (with id > {}), using {} ms", pc, maxId, System.currentTimeMillis() - ct); } - private void checkAndIndexIssues() throws GiteeException { + /** + * check and index issues data + * + * @param enterprise + * @throws GiteeException get issues data error + */ + private void checkAndIndexIssues(Enterprise enterprise) throws GiteeException { long ct = System.currentTimeMillis(); int pc = 0; Issue lastIssue = (Issue) QueryFactory.ISSUE().getLastestObject(); int maxId = lastIssue == null ? 0 : Math.toIntExact(lastIssue.getId()); - List issueList = GiteeApi.getInstance().getIssues(maxId); - for (com.gitee.kooder.models.gitee.Issue issue : issueList) { - QueueTask.add(Constants.TYPE_ISSUE, new Issue(issue)); + List issueList = GiteeApi.getInstance().getIssues(maxId); + for (com.gitee.kooder.gitee.Issue issue : issueList) { + Issue kooderIssue = issue.toKooderIssue(); + kooderIssue.setEnterprise(new Relation(enterprise.getId(), enterprise.getName(), enterprise.getUrl())); + QueueTask.add(Constants.TYPE_ISSUE, kooderIssue); pc++; } diff --git a/core/src/main/java/com/gitee/kooder/models/GiteeWebHookEvent.java b/indexer/src/main/java/com/gitee/kooder/gitee/GiteeWebHookEvent.java similarity index 41% rename from core/src/main/java/com/gitee/kooder/models/GiteeWebHookEvent.java rename to indexer/src/main/java/com/gitee/kooder/gitee/GiteeWebHookEvent.java index b16097ba0c5030fb547b5d38c0dcc2f443bf0295..7c6942c7f37524604e573b65e94204a9b10caf53 100644 --- a/core/src/main/java/com/gitee/kooder/models/GiteeWebHookEvent.java +++ b/indexer/src/main/java/com/gitee/kooder/gitee/GiteeWebHookEvent.java @@ -1,4 +1,19 @@ -package com.gitee.kooder.models; +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.gitee.kooder.gitee; import java.util.Arrays; @@ -7,7 +22,7 @@ import java.util.Arrays; */ public enum GiteeWebHookEvent { - REPO_HOO("Repo Hook"), + REPO_HOOK("Repo Hook"), PUSH_HOOK("Push Hook"), ISSUE_HOOK("Issue Hook"); diff --git a/core/src/main/java/com/gitee/kooder/models/gitee/Issue.java b/indexer/src/main/java/com/gitee/kooder/gitee/Issue.java similarity index 54% rename from core/src/main/java/com/gitee/kooder/models/gitee/Issue.java rename to indexer/src/main/java/com/gitee/kooder/gitee/Issue.java index 7d34d07a83a02e4ed37ec74f8581745b206bf3b7..1c62ef20605459ab619b846e6030e1ec140f82c5 100644 --- a/core/src/main/java/com/gitee/kooder/models/gitee/Issue.java +++ b/indexer/src/main/java/com/gitee/kooder/gitee/Issue.java @@ -1,7 +1,24 @@ -package com.gitee.kooder.models.gitee; +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.gitee.kooder.gitee; import com.fasterxml.jackson.annotation.JsonFormat; +import com.gitee.kooder.models.Relation; +import java.util.ArrayList; import java.util.Date; import java.util.List; @@ -16,27 +33,40 @@ public class Issue { public static final String STATE_REJECTED = "rejected"; private Integer id; - private String htmlUrl; - private String state; - private String title; - private String body; - private List labels; - private User user; - private Repository repository; @JsonFormat(pattern = "yyyy-MM-dd'T'HH:mm:ssX") private Date createdAt; - @JsonFormat(pattern = "yyyy-MM-dd'T'HH:mm:ssX") private Date updatedAt; + /** + * Turn to kooder issue + * @return + */ + public com.gitee.kooder.models.Issue toKooderIssue() { + com.gitee.kooder.models.Issue iss = new com.gitee.kooder.models.Issue(); + iss.setId(getId()); + iss.setIdent(this.getRepository().getId() + "_" + this.getId()); + iss.setRepository(new Relation(this.getRepository().getId(), this.getRepository().getName(), this.getRepository().getUrl())); + iss.setOwner(new Relation(this.getUser().getId(), this.getUser().getName(), this.getUser().getHtmlUrl())); + iss.setTitle(this.getTitle()); + iss.setDescription(this.getBody()); + iss.setUrl(this.getHtmlUrl()); + iss.setLabels(new ArrayList<>(this.getLabels())); + iss.setCreatedAt(this.getCreatedAt().getTime()); + iss.setUpdatedAt(this.getUpdatedAt().getTime()); + iss.setState((STATE_OPEN.equals(this.getState()) || STATE_PROGRESSING.equals(this.getState())) + ?com.gitee.kooder.models.Issue.STATE_OPENED:com.gitee.kooder.models.Issue.STATE_CLOSED); + return iss; + } + public Integer getId() { return id; } diff --git a/core/src/main/java/com/gitee/kooder/models/gitee/IssueWebHook.java b/indexer/src/main/java/com/gitee/kooder/gitee/IssueWebHook.java similarity index 39% rename from core/src/main/java/com/gitee/kooder/models/gitee/IssueWebHook.java rename to indexer/src/main/java/com/gitee/kooder/gitee/IssueWebHook.java index e784439cde448dd5d23e8fce25153935b019a8e7..88831b4b58fe11369642875013c1ef93ed1ee435 100644 --- a/core/src/main/java/com/gitee/kooder/models/gitee/IssueWebHook.java +++ b/indexer/src/main/java/com/gitee/kooder/gitee/IssueWebHook.java @@ -1,4 +1,19 @@ -package com.gitee.kooder.models.gitee; +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.gitee.kooder.gitee; /** * @author zhanggx diff --git a/indexer/src/main/java/com/gitee/kooder/gitee/PushWebHook.java b/indexer/src/main/java/com/gitee/kooder/gitee/PushWebHook.java new file mode 100644 index 0000000000000000000000000000000000000000..0f53b095961c9507bb6e43af73223db146043406 --- /dev/null +++ b/indexer/src/main/java/com/gitee/kooder/gitee/PushWebHook.java @@ -0,0 +1,32 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.gitee.kooder.gitee; + +/** + * @author zhanggx + */ +public class PushWebHook { + + private Repository repository; + + public Repository getRepository() { + return repository; + } + + public void setRepository(Repository repository) { + this.repository = repository; + } +} diff --git a/core/src/main/java/com/gitee/kooder/models/gitee/RepoWebHook.java b/indexer/src/main/java/com/gitee/kooder/gitee/RepoWebHook.java similarity index 45% rename from core/src/main/java/com/gitee/kooder/models/gitee/RepoWebHook.java rename to indexer/src/main/java/com/gitee/kooder/gitee/RepoWebHook.java index 2f0021699b5ffd843c6cfd169ba663aa268e4b8b..37f58d742d4cb4a74594a331f7a334bcb718f577 100644 --- a/core/src/main/java/com/gitee/kooder/models/gitee/RepoWebHook.java +++ b/indexer/src/main/java/com/gitee/kooder/gitee/RepoWebHook.java @@ -1,4 +1,19 @@ -package com.gitee.kooder.models.gitee; +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.gitee.kooder.gitee; /** * @author zhanggx diff --git a/core/src/main/java/com/gitee/kooder/models/gitee/Repository.java b/indexer/src/main/java/com/gitee/kooder/gitee/Repository.java similarity index 59% rename from core/src/main/java/com/gitee/kooder/models/gitee/Repository.java rename to indexer/src/main/java/com/gitee/kooder/gitee/Repository.java index b7930b338cbf45c88a236010b03723c672bbe044..17da1fcacf976795a4f74a6012d6a5f033933170 100644 --- a/core/src/main/java/com/gitee/kooder/models/gitee/Repository.java +++ b/indexer/src/main/java/com/gitee/kooder/gitee/Repository.java @@ -1,8 +1,26 @@ -package com.gitee.kooder.models.gitee; +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.gitee.kooder.gitee; import com.fasterxml.jackson.annotation.JsonFormat; import com.fasterxml.jackson.annotation.JsonProperty; +import com.gitee.kooder.core.Constants; +import com.gitee.kooder.models.Relation; +import java.util.Collections; import java.util.Date; /** @@ -11,13 +29,10 @@ import java.util.Date; public class Repository { private Integer id; - private String name; - private String url; - private String htmlUrl; - + private String gitHttpUrl; private String description; /** @@ -37,13 +52,9 @@ public class Repository { */ @JsonProperty("public") private Boolean isPublic; - private String license; - private String language; - private Integer stargazersCount; - private Integer forksCount; @JsonFormat(pattern = "yyyy-MM-dd'T'HH:mm:ssX") @@ -54,6 +65,32 @@ public class Repository { private User owner; + /** + * Turn to kooder repository + * @return + */ + public com.gitee.kooder.models.Repository toKooderRepository() { + com.gitee.kooder.models.Repository repo = new com.gitee.kooder.models.Repository(); + repo.setId(this.getId()); + repo.setName(this.getName()); + repo.setDescription(this.getDescription()); + repo.setUrl(this.getGitHttpUrl() == null ? this.getHtmlUrl() : this.getGitHttpUrl()); + repo.setOwner(new Relation(this.getOwner().getId(), this.getOwner().getName(), this.getOwner().getHtmlUrl())); + repo.setVisibility(this.getPrivate() ? Constants.VISIBILITY_PRIVATE : this.getInternal() ? Constants.VISIBILITY_INTERNAL : Constants.VISIBILITY_PUBLIC); + repo.setLicense(this.getLicense()); + repo.setLang(this.getLanguage()); + repo.setReadme(null); + repo.setFork(0); + repo.setTags(Collections.emptyList()); + repo.setStarsCount(this.getStargazersCount()); + repo.setForksCount(this.getForksCount()); + repo.setCreatedAt(this.getCreatedAt().getTime()); + repo.setUpdatedAt(this.getUpdatedAt().getTime()); + repo.setBlock(Constants.REPO_BLOCK_NO); + return repo; + } + + public Integer getId() { return id; } @@ -86,6 +123,14 @@ public class Repository { this.htmlUrl = htmlUrl; } + public String getGitHttpUrl() { + return gitHttpUrl; + } + + public void setGitHttpUrl(String gitHttpUrl) { + this.gitHttpUrl = gitHttpUrl; + } + public String getDescription() { return description; } diff --git a/core/src/main/java/com/gitee/kooder/models/gitee/User.java b/indexer/src/main/java/com/gitee/kooder/gitee/User.java similarity index 44% rename from core/src/main/java/com/gitee/kooder/models/gitee/User.java rename to indexer/src/main/java/com/gitee/kooder/gitee/User.java index 3373064a629a889fe2b9c9dbe9786f8fcb9c74e6..febdaa600b339031efad1f1c8987bfd3eddc3932 100644 --- a/core/src/main/java/com/gitee/kooder/models/gitee/User.java +++ b/indexer/src/main/java/com/gitee/kooder/gitee/User.java @@ -1,4 +1,19 @@ -package com.gitee.kooder.models.gitee; +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.gitee.kooder.gitee; /** * @author zhanggx diff --git a/indexer/src/main/java/com/gitee/kooder/indexer/FetchTaskThread.java b/indexer/src/main/java/com/gitee/kooder/indexer/FetchTaskThread.java index 24597e888aab19413055e0688e976abe23a80c60..69db8c4ec53c4798bdb7ded7b8f42d029d51ecf5 100644 --- a/indexer/src/main/java/com/gitee/kooder/indexer/FetchTaskThread.java +++ b/indexer/src/main/java/com/gitee/kooder/indexer/FetchTaskThread.java @@ -1,6 +1,22 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.indexer; import com.gitee.kooder.code.*; +import com.gitee.kooder.core.Constants; import com.gitee.kooder.core.GiteeSearchConfig; import com.gitee.kooder.models.CodeRepository; import com.gitee.kooder.models.Searchable; @@ -11,8 +27,10 @@ import com.gitee.kooder.storage.StorageFactory; import com.gitee.kooder.utils.BatchTaskRunner; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.math.NumberUtils; +import org.apache.lucene.document.LongPoint; import org.apache.lucene.facet.taxonomy.TaxonomyWriter; import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.search.Query; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -46,7 +64,7 @@ public class FetchTaskThread extends Thread { public void run() { while(!this.isInterrupted()) { final AtomicInteger taskCount = new AtomicInteger(0); - BatchTaskRunner.execute(provider.types(), 1, types -> { + BatchTaskRunner.execute(provider.getAllTypes(), 1, types -> { for(String type : types) { List tasks = provider.queue(type).pop(batch_fetch_count); if(tasks != null && tasks.size() > 0) { @@ -135,8 +153,15 @@ public class FetchTaskThread extends Thread { for(Searchable obj : task.getObjects()) { CodeRepository repo = RepositoryManager.INSTANCE.get(obj.getId()); if (repo != null) { - RepositoryFactory.getProvider(repo.getScm()).delete(repo); - RepositoryManager.INSTANCE.delete(repo.getId()); + + Query query = LongPoint.newExactQuery(Constants.FIELD_REPO_ID, repo.getId()); + try { + writer.deleteDocuments(query); // Delete SourceCode indexes + RepositoryFactory.getProvider(repo.getScm()).delete(repo); // Delete temp project git directory + RepositoryManager.INSTANCE.delete(repo.getId()); // Delete project metadata + } catch (IOException e) { + log.warn("Failed to delete code-repository id = " + repo.getId(), e); + } } } } diff --git a/indexer/src/main/java/com/gitee/kooder/indexer/GiteaIndexThread.java b/indexer/src/main/java/com/gitee/kooder/indexer/GiteaIndexThread.java index 9547a84540f00ed0f9260620bfdd824efeec9186..d5b064b387f90263f73f9f6ef403978d8d1e5da3 100644 --- a/indexer/src/main/java/com/gitee/kooder/indexer/GiteaIndexThread.java +++ b/indexer/src/main/java/com/gitee/kooder/indexer/GiteaIndexThread.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.indexer; import com.gitee.kooder.core.GiteeSearchConfig; diff --git a/indexer/src/main/java/com/gitee/kooder/indexer/Gitlab.java b/indexer/src/main/java/com/gitee/kooder/indexer/Gitlab.java index 52fc0468263c3a9a0a152860b3f96d5322c8b0ce..85429c02ab8f08cc6339dbaadfd8e55b09a97ce6 100644 --- a/indexer/src/main/java/com/gitee/kooder/indexer/Gitlab.java +++ b/indexer/src/main/java/com/gitee/kooder/indexer/Gitlab.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.indexer; import com.gitee.kooder.core.GiteeSearchConfig; diff --git a/indexer/src/main/java/com/gitee/kooder/indexer/GitlabIndexThread.java b/indexer/src/main/java/com/gitee/kooder/indexer/GitlabIndexThread.java index ee50a97fb05a956792bfc67bb44e9e31e40c4241..6c744d6dac5ed44c7dd0bceca4485e5cb2ca6abb 100644 --- a/indexer/src/main/java/com/gitee/kooder/indexer/GitlabIndexThread.java +++ b/indexer/src/main/java/com/gitee/kooder/indexer/GitlabIndexThread.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.indexer; import com.gitee.kooder.models.CodeRepository; @@ -132,6 +147,7 @@ public class GitlabIndexThread extends Thread { QueueTask.add(Constants.TYPE_REPOSITORY, repo); //index code CodeRepository codes = new CodeRepository(); + codes.setEnterprise(0); //Gitlab doesn't support enterprise codes.setId(p.getId()); codes.setScm(CodeRepository.SCM_GIT); codes.setName(p.getName()); diff --git a/indexer/src/main/java/com/gitee/kooder/indexer/PathImporter.java b/indexer/src/main/java/com/gitee/kooder/indexer/PathImporter.java index 3ef0ca99daa0eda2af884e5658566e047078744b..cd2403da4ecb93e2d9eb6958ffe91c908c5ddee5 100644 --- a/indexer/src/main/java/com/gitee/kooder/indexer/PathImporter.java +++ b/indexer/src/main/java/com/gitee/kooder/indexer/PathImporter.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.indexer; import com.gitee.kooder.queue.QueueTask; diff --git a/indexer/src/main/java/com/gitee/kooder/indexer/RepositoryIndexer.java b/indexer/src/main/java/com/gitee/kooder/indexer/RepositoryIndexer.java index 166028f1cba83abb57e4348ca3f73ce25db5b007..ea0eed48fc5204e4ffb221ebe9bad39191fff409 100644 --- a/indexer/src/main/java/com/gitee/kooder/indexer/RepositoryIndexer.java +++ b/indexer/src/main/java/com/gitee/kooder/indexer/RepositoryIndexer.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.indexer; import org.eclipse.jgit.api.Git; diff --git a/indexer/src/main/java/com/gitee/kooder/indexer/ServerDaemon.java b/indexer/src/main/java/com/gitee/kooder/indexer/ServerDaemon.java index 12c28adcf273a139937053a7fa9e47174bb52411..cbac80193fd047552dc462b7afa919eafcba787b 100644 --- a/indexer/src/main/java/com/gitee/kooder/indexer/ServerDaemon.java +++ b/indexer/src/main/java/com/gitee/kooder/indexer/ServerDaemon.java @@ -1,3 +1,18 @@ +/** + * Copyright (c) 2021, OSChina (oschina.net@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.gitee.kooder.indexer; import org.apache.commons.daemon.Daemon; diff --git a/pom.xml b/pom.xml index afdf3608654fdae241f33f546ea36f16c02851ad..de02ef181a3b3e70042d29c4b149f05e7cb99516 100644 --- a/pom.xml +++ b/pom.xml @@ -35,8 +35,8 @@ maven-compiler-plugin 3.8.0 - 11 - 11 + 8 + 8 ${project.build.sourceEncoding} diff --git a/readme.md b/readme.md index 15e1a0582037996766c9922edec57e6138a6a0ab..4702edb2a2efe006eba11ea3e3c8f10edad1bd67 100644 --- a/readme.md +++ b/readme.md @@ -38,7 +38,7 @@ Kooder 服务包含两个模块,分别是 gateway 和 indexer(默认配置 1.依赖 -* openjdk >= 11 +* openjdk >= 8 * maven > 3 2.下载代码 @@ -66,18 +66,24 @@ $ cd kooder `gitlab.url` 访问 Gitlab 的首页 `gitlab.personal_access_token` Gitlab 管理员账号 root 的 Personal Access Token -更多配置项请看 [configuration.md](configuration.md) +更多配置项请看 [configuration.md](docs/configuration.md) 3. 构建并运行 ``` $ cd Kooder $ mvn install +### 给执行脚本添加权限 +$ chmod +x bin/*.sh ### 启动 gateway $ bin/gateway.sh ### 浏览器访问 http://localhost:8080 ``` +### Kooder Search API + +@see [API.md](docs/API.md) + ### Docker安装 #### docker-compose diff --git a/readme_en.md b/readme_en.md index cb0d511a9e368c1e714ded589d40e3c62de6a343..4d18b2f33982775ffbbff251884d91a3459cfc8e 100644 --- a/readme_en.md +++ b/readme_en.md @@ -33,7 +33,7 @@ Indexer: 1.Dependencies -* openjdk >= 11 +* openjdk >= 8 * maven > 3 2.Download source code @@ -52,7 +52,7 @@ Config kooder's url `http.url`. It will be injected into Git service as the webh ``` http.url = http://:8080 ``` -Click here to see more config options [configuration.md](configuration.md) +Click here to see more config options [configuration.md](docs/configuration.md) ### Install Docker Dependencies