Files
VisualSapfor/src/TestingSystem/Common/TestingServer.java

628 lines
30 KiB
Java
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package TestingSystem.Common;
import Common.Constants;
import Common.Database.DBObject;
import Common.Global;
import Common.Utils.Utils;
import GlobalData.Account.Account;
import ProjectData.LanguageName;
import Repository.Component.Sapfor.Sapfor;
import Repository.EmailMessage;
import Repository.RepositoryRefuseException;
import Repository.RepositoryServer;
import Repository.Server.ServerCode;
import Repository.Server.ServerExchangeUnit_2021;
import TestingSystem.Common.Group.Group;
import TestingSystem.Common.Test.Test;
import TestingSystem.Common.Test.TestType;
import TestingSystem.Common.TestingPackageToKill.TestingPackageToKill;
import TestingSystem.DVM.DVMPackage.DVMPackage;
import TestingSystem.DVM.DVMTestingPlanner;
import TestingSystem.SAPFOR.SapforConfiguration.SapforConfiguration;
import TestingSystem.SAPFOR.SapforConfigurationCommand.SapforConfigurationCommand;
import TestingSystem.SAPFOR.SapforPackage.SapforPackage;
import TestingSystem.SAPFOR.SapforTestingPlanner;
import TestingSystem.SAPFOR.ServerSapfor.ServerSapfor;
import Visual_DVM_2021.Passes.All.DownloadRepository;
import Visual_DVM_2021.Passes.All.ZipFolderPass;
import Visual_DVM_2021.Passes.PassCode_2021;
import Visual_DVM_2021.Passes.Pass_2021;
import javafx.util.Pair;
import org.apache.commons.io.FileUtils;
import javax.swing.Timer;
import java.io.File;
import java.nio.file.Paths;
import java.util.*;
public class TestingServer extends RepositoryServer<TestsDatabase> {
/*
@Override
protected void beforePublishAction(DBObject object) throws Exception {
if (object instanceof Group) {
Group group = (Group) object;
if (db.groups.containsGroupWithDescription(group.description))
throw new RepositoryRefuseException("Уже существует группа с описанием " +
Utils.Brackets(group.description.toLowerCase()));
} else if (object instanceof Test) {
Test test = (Test) object;
if (db.tests.containsTestWithDescription(test.description))
throw new RepositoryRefuseException("Уже существует тест с описанием " +
Utils.Brackets(test.description.toLowerCase()));
}
}
*/
@Override
public void afterPublishAction(DBObject object) throws Exception {
if (object instanceof Test) {
Test test = (Test) object;
if (!test.unpackProjectOnServer()) {
db.Delete(test);
throw new RepositoryRefuseException(
"Не удалось прикрепить проект к тесту с id " + test.id
+ "\nТест будет удален"
);
}
} else if (object instanceof DVMPackage) {
DVMPackage dvmPackage = (DVMPackage) object;
//--
Utils.CheckAndCleanDirectory(dvmPackage.getLocalWorkspace());
//--
dvmPackage.saveJson();
dvmPackage.package_json = null; // объект больше не нужен.
} else if (object instanceof SapforPackage) {
((SapforPackage) object).init();
}
}
@Override
public void afterDeleteAction(DBObject object) throws Exception {
if (object instanceof Test) {
Test test = (Test) object;
Utils.forceDeleteWithCheck(test.getArchive());
Utils.forceDeleteWithCheck(test.getServerPath());
} else if (object instanceof Group) {
Group group = (Group) object;
Vector<Test> tests = new Vector<>();
for (Test group_test : db.tests.Data.values()) {
if (group_test.group_id == group.id)
tests.add(group_test);
}
for (Test group_test : tests) {
db.Delete(group_test);
Utils.forceDeleteWithCheck(group_test.getArchive());
Utils.forceDeleteWithCheck(group_test.getServerPath());
}
} else if (object instanceof ServerSapfor) {
Utils.forceDeleteWithCheck(
new File(
((ServerSapfor) object).home_path
)
);
} else if (object instanceof SapforConfiguration) {
SapforConfiguration sapforConfiguration = (SapforConfiguration) object;
Vector<SapforConfigurationCommand> commands = new Vector<>();
for (SapforConfigurationCommand command : db.sapforConfigurationCommands.Data.values()) {
if (command.sapforconfiguration_id == sapforConfiguration.id)
commands.add(command);
}
for (SapforConfigurationCommand command : commands) {
db.Delete(command);
}
} else if (object instanceof DVMPackage) {
DVMPackage dvmPackage = (DVMPackage) object;
File workspace = dvmPackage.getLocalWorkspace();
Utils.forceDeleteWithCheck(workspace);
} else if (object instanceof SapforPackage) {
SapforPackage sapforPackage = (SapforPackage) object;
File workspace = sapforPackage.getLocalWorkspace();
Utils.forceDeleteWithCheck(workspace);
}
}
//-->>>
public TestingServer() {
super(TestsDatabase.class);
}
//основа
@Override
public int getPort() {
return Global.properties.TestingServerPort;
}
//---
@Override
protected void startAdditionalThreads() {
testingThread.start();
}
protected DVMTestingPlanner DVMTestingPlanner = new DVMTestingPlanner();
protected SapforTestingPlanner sapforTestingPlanner = new SapforTestingPlanner();
//--
protected Thread testingThread = new Thread(() -> {
while (true) {
DVMTestingPlanner.Perform();
sapforTestingPlanner.Perform();
}
});
//------>>>
//------>>>
public static Timer checkTimer = null;
public static void TimerOn() {
System.out.println("timer on");
checkTimer = new Timer(Global.properties.CheckTestingIntervalSeconds * 1000, e -> {
// Pass_2021.passes.get(PassCode_2021.SynchronizeTestsTasks).Do();
Pass_2021.passes.get(PassCode_2021.ActualizePackages).Do();
});
checkTimer.start();
}
public static void TimerOff() {
System.out.println("timer off");
if (checkTimer != null)
checkTimer.stop();
}
public static void ResetTimer() {
TimerOff();
TimerOn();
}
@Override
protected void Session() throws Exception {
Test test;
int test_id;
switch (code) {
case EmailSapforAssembly:
/*
Print("Сообщить о сборке SAPFOR для пользователя " + request.arg);
Vector<String> assembly_info = (Vector<String>) request.object;
File out = Paths.get(Global.RepoDirectory.getAbsolutePath(), Constants.SAPFOR_REPOSITORY_BIN, Constants.out_file).toFile();
File err = Paths.get(Global.RepoDirectory.getAbsolutePath(), Constants.SAPFOR_REPOSITORY_BIN, Constants.err_file).toFile();
Vector<String> targets = new Vector<>(Arrays.asList(Global.admins_mails));
EmailMessage message = new EmailMessage(
"Выполнена сборка системы SAPFOR",
"Версия: " + assembly_info.get(0) + "\n" + "Статус: " + assembly_info.get(1),
targets
);
Email(message, out, err);
response = new ServerExchangeUnit_2021(ServerCode.OK);
*/
break;
//------------------------------------------->>
case DownloadTest:
Print("Отправить клиенту тест " + request.arg);
test_id = Integer.parseInt(request.arg);
if (db.tests.containsKey(test_id)) {
test = db.tests.get(test_id);
response = new ServerExchangeUnit_2021(ServerCode.OK, "", Utils.packFile(test.getArchive()));
} else
throw new RepositoryRefuseException("Теста с именем " + request.arg + " не существует");
break;
case ReceiveTestsDatabase:
Print("Получить базу данных тестов");
response = new ServerExchangeUnit_2021(ServerCode.OK);
response.object = Utils.packFile(db.getFile());
break;
//---
case RefreshDVMTests:
Print("Синхронизировать репозиторий тестов ");
response = new ServerExchangeUnit_2021(ServerCode.OK);
RefreshDVMTests((Account) request.object, Integer.parseInt(request.arg));
break;
case GetFirstActiveDVMPackage:
Print("Получить первый активный пакет задач DVM");
GetFirstActiveDVMPackage();
break;
case DVMPackageNeedsKill:
Print("Проверить нуждается ли пакет DVM в убийстве");
DVMPackageNeedsKill();
break;
case UpdateActiveDVMPackages:
Print("Получить данные по пакетам DVM");
UpdateActiveDVMPackages();
break;
case GetFirstActiveSapforPackage:
Print("Получить первый активный пакет задач SAPFOR");
GetFirstActiveSapforPackage();
break;
case SapforPackageNeedsKill:
Print("Проверить нуждает ли пакет SAPFOR в убийстве");
SapforPackageNeedsKill();
break;
case UpdateActiveSapforPackages:
Print("Получить данные по пакетам Sapfor");
UpdateActiveSapforPackages();
break;
case DownloadDVMPackage:
Print("Загрузить пакет DVM");
DownloadDVMPackage();
break;
case DownloadDVMPackages:
Print("Загрузить пакеты DVM");
DownloadDVMPackages();
break;
case DownloadSapforPackage:
Print("Загрузить пакет SAPFOR");
DownloadSapforPackage();
break;
case InstallServerSapfor:
Print("Установить текущую версию SAPFOR для тестирования");
InstallServerSapfor();
break;
case ReplaceTestCode:
Print("Заменить код теста");
ReplaceTestCode();
break;
case ReplaceTestsCodes:
Print("Заменить код тестов");
ReplaceTestsCodes();
break;
default:
throw new RepositoryRefuseException("Неподдерживаемый код: " + code);
}
}
//->>
Pair<Group, Vector<File>> ConvertDirectoryToGroup(File src, LanguageName languageName, TestType
testType, Account account) throws Exception {
Group object = new Group();
Vector<File> groupFiles = null; //транспорт.
//->>
object.description = src.getName();
object.language = languageName;
object.type = testType;
object.sender_name = account.name;
object.sender_address = account.email;
//-->>
File[] files = src.listFiles(pathname ->
pathname.isFile()
&& !pathname.getName().equals("settings")
&& !pathname.getName().equals("test-analyzer.sh")
&& Utils.getExtension(pathname).startsWith(languageName.getDVMCompile()));
;
if (files != null) {
groupFiles = new Vector<>(Arrays.asList(files));
groupFiles.sort(Comparator.comparing(File::getName));
}
//->>
return new Pair<>(object, groupFiles);
}
public void RefreshDVMTests(Account account, int sapfor_id) throws Exception {
ServerSapfor sapfor = null;
if (!db.serverSapfors.containsKey(sapfor_id))
throw new RepositoryRefuseException("Версия SAPFOR с ключом " + sapfor_id + " не найдена.");
sapfor = db.serverSapfors.get(sapfor_id);
DownloadRepository downloadRepository = new DownloadRepository();
if (!downloadRepository.Do())
throw new RepositoryRefuseException("Не удалось обновить репозиторий");
//-->>
Vector<Pair<Group, Vector<File>>> groups = new Vector<>();
LinkedHashMap<Group, Vector<Test>> res = new LinkedHashMap<>();
File testsSrc = Paths.get(
Global.RepoDirectory.getAbsolutePath(),
"dvm", "tools", "tester", "trunk", "test-suite").toFile();
LanguageName[] supportedLanguages = new LanguageName[]{LanguageName.fortran, LanguageName.c};
for (LanguageName languageName : supportedLanguages) {
for (TestType testType : TestType.values()) {
File groupsSrc = null;
switch (testType) {
case Correctness:
String languageSrcName = null;
switch (languageName) {
case fortran:
languageSrcName = "Fortran";
break;
case c:
languageSrcName = "C";
break;
}
if (languageSrcName != null) {
groupsSrc = Paths.get(testsSrc.getAbsolutePath(), "Correctness", languageSrcName).toFile();
File[] groupsDirs = groupsSrc.listFiles(File::isDirectory);
if (groupsDirs != null) {
for (File groupDir : groupsDirs)
groups.add(ConvertDirectoryToGroup(groupDir, languageName, testType, account));
}
}
break;
case Performance:
File groupDir = Paths.get(testsSrc.getAbsolutePath(), "Performance").toFile();
groups.add(ConvertDirectoryToGroup(groupDir, languageName, testType, account));
break;
}
}
}
groups.sort(Comparator.comparing(o -> o.getKey().description));
//-теперь создать тесты.
System.out.println("найдено " + groups.size() + " групп");
//--
for (Pair<Group, Vector<File>> p : groups) {
Group group = p.getKey();
//-
db.Insert(group);
Vector<Test> testsIds = new Vector<>();
res.put(group, testsIds);
//-
Vector<File> files = p.getValue();
if (!files.isEmpty()) {
//->>
for (File file : files) {
System.out.println("Создание теста " + file.getName());
Test test = new Test();
test.description = Utils.getNameWithoutExtension(file.getName()) + "_" + group.language.getDVMCompile();
test.sender_name = account.name;
test.sender_address = account.email;
test.group_id = group.id;
test.files = file.getName();
db.Insert(test);
testsIds.add(test);
//->>
File testDirectory = new File(Global.TestsDirectory, String.valueOf(test.id));
Utils.CheckAndCleanDirectory(testDirectory);
File testFile = Paths.get(testDirectory.getAbsolutePath(), file.getName()).toFile();
FileUtils.copyFile(file, testFile);
//----
//архивация.
File archive = test.getArchive();
ZipFolderPass zip = new ZipFolderPass();
zip.Do(testDirectory.getAbsolutePath(), archive.getAbsolutePath());
//---
//Определение размерности
switch (group.language) {
case fortran:
// временная папка для анализа. чтобы не засорять нормальную.
File tempProject = Utils.getTempFileName("test");
FileUtils.forceMkdir(tempProject);
FileUtils.copyDirectory(testDirectory, tempProject);
//--
if (Sapfor.getMinMaxDim(Sapfor.getTempCopy(new File(sapfor.call_command)), tempProject, test)) {
db.Update(test);
} else
throw new RepositoryRefuseException("Не удалось определить размерность теста " + Utils.Brackets(test.description));
break;
case c:
test.max_dim = Utils.getCTestMaxDim(testFile);
db.Update(test);
break;
}
}
}
}
}
//-------------------------------------------------------------------------------------->>>
void GetFirstActiveDVMPackage() throws Exception {
response = new ServerExchangeUnit_2021(ServerCode.OK);
response.object = null;
DVMPackage dvmPackage = db.getFirstActiveDVMPackage();
if (dvmPackage != null) {
//нужно вернуть копию объекта с иным адресом!!
response.object = new DVMPackage(dvmPackage);
}
}
private void GetFirstActiveSapforPackage() throws Exception {
response = new ServerExchangeUnit_2021(ServerCode.OK);
response.object = null;
SapforPackage sapforPackage = db.getFirstActiveSapforPackage();
if (sapforPackage != null) {
//нужно вернуть копию объекта с иным адресом!!
response.object = new SapforPackage(sapforPackage);
}
}
//---
void UpdateActiveDVMPackages() throws Exception {
response = new ServerExchangeUnit_2021(ServerCode.OK);
Vector<Pair<Integer, Long>> keys_pairs = (Vector<Pair<Integer, Long>>) request.object;
Vector<DVMPackage> res = new Vector<>();
//--
for (Pair<Integer, Long> p : keys_pairs) {
if (db.dvmPackages.containsKey(p.getKey())) {
DVMPackage actual = db.dvmPackages.get(p.getKey());
if (actual.ChangeDate != p.getValue())
res.add(new DVMPackage(actual));
}
}
response.object = res;
}
private void UpdateActiveSapforPackages() {
response = new ServerExchangeUnit_2021(ServerCode.OK);
Vector<Pair<Integer, Long>> keys_pairs = (Vector<Pair<Integer, Long>>) request.object;
Vector<SapforPackage> res = new Vector<>();
//--
for (Pair<Integer, Long> p : keys_pairs) {
if (db.sapforPackages.containsKey(p.getKey())) {
SapforPackage actual = db.sapforPackages.get(p.getKey());
if (actual.ChangeDate != p.getValue())
res.add(new SapforPackage(actual));
}
}
response.object = res;
}
private void DVMPackageNeedsKill() {
response = new ServerExchangeUnit_2021(ServerCode.OK);
response.object = Constants.Nan;
int packageId = (int) request.object;
for (TestingPackageToKill packageToKill : db.testingPackagesToKill.Data.values()) {
if ((packageToKill.packageId == packageId) && (packageToKill.type == 0)) {
response.object = packageToKill.id;
break;
}
}
}
private void SapforPackageNeedsKill() throws Exception {
response = new ServerExchangeUnit_2021(ServerCode.OK);
response.object = Constants.Nan;
int packageId = (int) request.object;
for (TestingPackageToKill packageToKill : db.testingPackagesToKill.Data.values()) {
if ((packageToKill.packageId == packageId) && (packageToKill.type == 1)) {
response.object = packageToKill.id;
break;
}
}
}
private void DownloadDVMPackage() throws Exception {
int dvmPackage_id = (int) request.object;
if (!db.dvmPackages.containsKey(dvmPackage_id))
throw new RepositoryRefuseException("Не найдено пакета тестирования DVM с ключом " + dvmPackage_id);
response = new ServerExchangeUnit_2021(ServerCode.OK);
DVMPackage dvmPackage = db.dvmPackages.get(dvmPackage_id);
File workspace = dvmPackage.getLocalWorkspace();
File results_zip = new File(workspace, "results.zip");
File package_json = dvmPackage.getJsonFile();
response.object = new Pair(Utils.packFile(results_zip), Utils.packFile(package_json));
}
private void DownloadDVMPackages() throws Exception {
Vector<Integer> ids = (Vector<Integer>) request.object;
Vector<Pair<Integer, Pair<byte[], byte[]>>> res = new Vector<>();
for (int dvmPackage_id : ids) {
if (!db.dvmPackages.containsKey(dvmPackage_id))
throw new RepositoryRefuseException("Не найдено пакета тестирования DVM с ключом " + dvmPackage_id);
DVMPackage dvmPackage = db.dvmPackages.get(dvmPackage_id);
File workspace = dvmPackage.getLocalWorkspace();
File results_zip = new File(workspace, "results.zip");
File package_json = dvmPackage.getJsonFile();
res.add(new Pair<>(dvmPackage_id, new Pair(Utils.packFile(results_zip), Utils.packFile(package_json))));
}
response = new ServerExchangeUnit_2021(ServerCode.OK);
response.object = res;
}
private void DownloadSapforPackage() throws Exception {
int sapforPackage_id = (int) request.object;
if (!db.sapforPackages.containsKey(sapforPackage_id))
throw new RepositoryRefuseException("Не найдено пакета тестирования SAPFOR с ключом " + sapforPackage_id);
response = new ServerExchangeUnit_2021(ServerCode.OK);
SapforPackage sapforPackage = db.sapforPackages.get(sapforPackage_id);
File workspace = sapforPackage.getLocalWorkspace();
File results_zip = Utils.getTempFileName("results");
ZipFolderPass zipFolderPass = new ZipFolderPass();
zipFolderPass.Do(workspace.getAbsolutePath(), results_zip.getAbsolutePath());
if (results_zip.exists())
response.object = Utils.packFile(results_zip);
else
throw new RepositoryRefuseException("Не удалось заархивировать пакет тестирования SAPFOR с ключом " + sapforPackage_id);
}
private void InstallServerSapfor() throws Exception {
int max_version = Constants.Nan;
int current_version = Constants.Nan;
//--
for (ServerSapfor sapfor : db.serverSapfors.Data.values()) {
int version = -1;
try {
version = Integer.parseInt(sapfor.version);
} catch (Exception ex) {
ex.printStackTrace();
}
if (version > max_version) max_version = version;
}
System.out.println("max Sapfor version is " + max_version);
//--
File testingSystemHome = new File(Global.Home);
File repo = new File(testingSystemHome, "Repo");
File repoSapforHome = Paths.get(repo.getAbsolutePath(), Constants.SAPFOR_REPOSITORY_BIN).toFile();
File repo_bin = new File(repoSapforHome, "Sapfor_F");
//--
System.out.println("Синхронизация ветви DVM...");
Utils.startScript(repo, repo, "dvm_checkout",
"svn checkout " +
Constants.REPOSITORY_AUTHENTICATION +
" " + Constants.DVM_REPOSITORY + " 1>dvm_out.txt 2>dvm_err.txt\n").waitFor();
System.out.println("Синхронизация ветви SAPFOR...");
Utils.startScript(repo, repo, "spf_checkout",
"svn checkout " +
Constants.REPOSITORY_AUTHENTICATION +
" " + Constants.SAPFOR_REPOSITORY + " 1>spf_out.txt 2>spf_err.txt\n").waitFor();
//--
if (repo_bin.exists())
FileUtils.forceDelete(repo_bin);
//-
File versionFile = Paths.get(repo.getAbsolutePath(), "/sapfor/experts/Sapfor_2017/_src/Utils/version.h").toFile();
if (versionFile.exists()) {
List<String> data = FileUtils.readLines(versionFile);
for (String s : data) {
if (s.startsWith("#define VERSION_SPF ")) {
String[] version_data = s.split("\"");
if (version_data.length > 0) {
String version_s = version_data[1];
//-
try {
current_version = Integer.parseInt(version_s);
} catch (Exception ex) {
ex.printStackTrace();
}
//-
if (current_version == max_version) {
throw new RepositoryRefuseException("Версия SAPFOR " + max_version + " уже собрана!");
}
}
}
}
}
//--
System.out.println("Сборка SAPFOR...");
Utils.startScript(repoSapforHome, repoSapforHome, "build_sapfor",
"cmake ../ 1>out.txt 2>err.txt\nmake -j 14 1>>out.txt 2>>err.txt\n").waitFor();
//--
System.out.println("DONE");
File repoSapfor = new File(repoSapforHome, "Sapfor_F");
System.out.println("Result file is " + Utils.Brackets(repoSapfor.getAbsolutePath()));
if (repoSapfor.exists()) {
System.out.println("assembly found!");
response = new ServerExchangeUnit_2021(ServerCode.OK);
File sapforsDirectory = new File(testingSystemHome, "Sapfors");
//создать папку. Для того чтобы скопировать из репозитория.
File sapforHome = new File(sapforsDirectory, Utils.getDateName("sapfor"));
sapforHome.mkdir();
File sapforBin = new File(sapforHome, "Sapfor_F");
FileUtils.copyFile(repo_bin, sapforBin);
sapforBin.setExecutable(true, false);
// //-->>>
ServerSapfor serverSapfor = new ServerSapfor();
serverSapfor.home_path = sapforHome.getAbsolutePath();
serverSapfor.call_command = sapforBin.getAbsolutePath();
serverSapfor.languageName = LanguageName.fortran;
serverSapfor.buildDate = new Date().getTime();
response.object = serverSapfor;
serverSapfor.version = String.valueOf(current_version);
//---
EmailSapforAssembly(current_version, true);
//---
} else {
//---
EmailSapforAssembly(current_version, false);
//---
throw new RepositoryRefuseException("Бинарный файл SAPFOR не найден!");
}
}
private void ReplaceTestCode() throws Exception {
Test test = (Test) request.object;
response = new ServerExchangeUnit_2021(ServerCode.OK);
//---
if (!test.unpackProjectOnServer()) {
db.Delete(test);
throw new RepositoryRefuseException(
"Не удалось прикрепить проект к тесту с id " + test.id
+ "\nТест будет удален"
);
} else db.Update(test); //обновить список файлов и размерность.
}
private void ReplaceTestsCodes() throws Exception {
Vector<Test> tests = (Vector<Test>) request.object;
response = new ServerExchangeUnit_2021(ServerCode.OK);
for (Test test : tests) {
if (!test.unpackProjectOnServer()) {
db.Delete(test);
throw new RepositoryRefuseException(
"Не удалось прикрепить проект к тесту с id " + test.id
+ "\nТест будет удален"
);
} else db.Update(test); //обновить список файлов и размерность.
}
}
private void EmailSapforAssembly(int version, boolean done) throws Exception {
String version_s = (version == Constants.Nan) ? "?" : String.valueOf(version);
String status = done ? "Успешно" : "С ошибками";
//-
File out = Paths.get(Global.RepoDirectory.getAbsolutePath(), Constants.SAPFOR_REPOSITORY_BIN, Constants.out_file).toFile();
File err = Paths.get(Global.RepoDirectory.getAbsolutePath(), Constants.SAPFOR_REPOSITORY_BIN, Constants.err_file).toFile();
Vector<String> targets = new Vector<>(Arrays.asList(Global.admins_mails));
EmailMessage message = new EmailMessage(
"Выполнена сборка системы SAPFOR",
"Версия: " + version_s + "\n" + "Статус: " + status,
targets
);
//-
Email(message, out, err);
}
}