M/VisualSapfor
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Доделаны процессы для разных машин

Browse Source
This commit is contained in:
02090095
2024-04-24 22:23:50 +03:00
parent 4d543cb94f
commit c6e5036b9c
12 changed files with 143 additions and 471 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
.idea/workspace.xml generated
View File

@@ -7,17 +7,17 @@
</component>
<component name="ChangeListManager">
<list default="true" id="e42177c3-2328-4b27-8a01-35779b2beb99" name="Default Changelist" comment="">
<change afterPath="$PROJECT_DIR$/src/TestingSystem/Common/MachineProcess/MachineProcessSet.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/properties" beforeDir="false" afterPath="$PROJECT_DIR$/properties" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/Common/GlobalProperties.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/Common/GlobalProperties.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/Common/Utils/Utils.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/Common/Utils/Utils.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/Repository/RepositoryServer.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/Repository/RepositoryServer.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/Repository/Server/ServerCode.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/Repository/Server/ServerCode.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/TestingSystem/Common/MachineProcess/MachineProcess.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/TestingSystem/Common/MachineProcess/MachineProcess.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/TestingSystem/Common/MachineProcess/MachineProcessDBTable.java" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/src/TestingSystem/Common/MachineProcess/MachineProcessSet.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/TestingSystem/Common/MachineProcess/MachineProcessSet.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/TestingSystem/Common/TestingPlanner.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/TestingSystem/Common/TestingPlanner.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/TestingSystem/Common/TestingServer.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/TestingSystem/Common/TestingServer.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/TestingSystem/Common/TestsDatabase.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/TestingSystem/Common/TestsDatabase.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/TestingSystem/DVM/DVMTestingPlanner.java" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/src/TestingSystem/DVM/MachineQueueSupervisor.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/TestingSystem/DVM/MachineQueueSupervisor.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/TestingSystem/SAPFOR/SapforTestingPlanner.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/TestingSystem/SAPFOR/SapforTestingPlanner.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/Visual_DVM_2021/Passes/Pass_2021.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/Visual_DVM_2021/Passes/Pass_2021.java" afterDir="false" />
</list>
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />

4
properties
View File

@@ -2,9 +2,9 @@
"Mode": "Normal",
"ServerAddress": "alex-freenas.ddns.net",
"ServerUserName": "testuser",
"ServerUserSHHPort": 23,
"ServerUserSHHPort": 2000,
"ComponentsServerPort": 7995,
"TestingServerPort": 7996,
"TestingServerPort": 7998,
"SocketTimeout": 5000,
"OldServer": false,
"SMTPHost": "smtp.mail.ru",

47
src/Common/GlobalProperties.java
View File

@@ -119,4 +119,51 @@ public class GlobalProperties extends Properties {
public File getFile() {
return Paths.get(System.getProperty("user.dir"), "properties").toFile();
}
//--
public GlobalProperties(){
}
public GlobalProperties(GlobalProperties p) {
super();
Mode=p.Mode;
ServerAddress = p.ServerAddress;
ServerUserName = p.ServerUserName;
ServerUserSHHPort = p.ServerUserSHHPort;
ComponentsServerPort = p.ComponentsServerPort;
TestingServerPort = p.TestingServerPort;
SocketTimeout = p.SocketTimeout;
OldServer = p.OldServer;
SMTPHost = p.SMTPHost;
SMTPPort = p.SMTPPort;
MailSocketPort = p.MailSocketPort;
collapseCredentials = p.collapseCredentials;
collapseFileGraphs = p.collapseFileGraphs;
collapseFileMessages = p.collapseFileMessages;
collapseProjectTrees = p.collapseProjectTrees;
BackupWorkspace = p.BackupWorkspace;
BackupHour = p.BackupHour;
BackupMinute = p.BackupMinute;
EmailAdminsOnStart = p.EmailAdminsOnStart;
AutoUpdateSearch = p.AutoUpdateSearch;
ConfirmPassesStart = p.ConfirmPassesStart;
ShowPassesDone = p.ShowPassesDone;
FocusPassesResult = p.FocusPassesResult;
GlobalDBName = p.GlobalDBName;
ProjectDBName = p.ProjectDBName;
BugReportsDBName = p.BugReportsDBName;
TestsDBName = p.TestsDBName;
ComponentsWindowWidth = p.ComponentsWindowWidth;
ComponentsWindowHeight = p.ComponentsWindowHeight;
VisualiserPath = p.VisualiserPath;
Sapfor_FPath = p.Sapfor_FPath;
Visualizer_2Path = p.Visualizer_2Path;
InstructionPath = p.InstructionPath;
PerformanceAnalyzerPath = p.PerformanceAnalyzerPath;
ComponentsBackUpsCount = p.ComponentsBackUpsCount;
TestingKernels = p.TestingKernels;
AutoCheckTesting = p.AutoCheckTesting;
CheckTestingIntervalSeconds = p.CheckTestingIntervalSeconds;
EmailOnTestingProgress = p.EmailOnTestingProgress;
eraseTestingWorkspaces = p.eraseTestingWorkspaces;
}
}

2
src/Common/Utils/Utils.java
View File

@@ -1157,7 +1157,7 @@ public class Utils {
for (int i = 1; i <= 10; ++i) {
System.out.println("Проверка " + i + " существования файла " + Utils.Brackets(file.getAbsolutePath()));
if (file.exists()) return true;
else sleep(5000);
else sleep(1000);
}
return false;
}

10
src/TestingSystem/Common/MachineProcess/MachineProcess.java
View File

@@ -19,15 +19,12 @@ public class MachineProcess extends DBObject {
public String userWorkspace = "";
public String testingSystemRoot = "";
public String serverName = "";
public MachineProcessState state = MachineProcessState.Inactive; //0 неактивен
//--
public MachineProcess() {
}
public MachineProcess(MachineProcess p) {
SynchronizeFields(p);
}
public MachineProcess(DVMPackage p) {
id = Utils.getDateName("machine_process");
machineAddress = p.machine_address;
machinePort = p.machine_port;
userName = p.user_name;
@@ -35,6 +32,7 @@ public class MachineProcess extends DBObject {
userWorkspace = p.user_workspace;
testingSystemRoot = Global.Home;
serverName = Global.testingServer.name;
id = Utils.getDateName(machineAddress+"_"+machinePort+"_"+userName);
}
@Override
public Object getPK() {
@@ -88,9 +86,8 @@ public class MachineProcess extends DBObject {
File supervisor = new File(workspace, "VisualSapfor.jar");
FileUtils.copyFile(src, supervisor);
//создание настроек
GlobalProperties properties = new GlobalProperties();
GlobalProperties properties = new GlobalProperties(Global.properties);
properties.Mode = Current.Mode.MachineQueue;
properties.TestingServerPort = 7996;
Utils.jsonToFile(properties, new File(workspace, "properties"));
Vector<String> args = new Vector<>();
args.add(Utils.DQuotes(machineAddress));
@@ -103,7 +100,8 @@ public class MachineProcess extends DBObject {
//--
Utils.startScript(workspace, workspace,
"start",
"java -jar VisualSapfor.jar " + String.join(" ", args)+" ");
"java -jar VisualSapfor.jar " +
String.join(" ", args)+" 1>out.txt 2>err.txt");
//---
} catch (Exception ex) {
ex.printStackTrace();

14
src/TestingSystem/Common/MachineProcess/MachineProcessSet.java
View File

@@ -1,5 +1,6 @@
package TestingSystem.Common.MachineProcess;
import Common.Database.DataSet;
import TestingSystem.DVM.DVMPackage.DVMPackage;
import java.util.LinkedHashMap;
public class MachineProcessSet extends DataSet<String, MachineProcess> {
@@ -12,4 +13,17 @@ public class MachineProcessSet extends DataSet<String, MachineProcess> {
res.put(process.getUniqueKey(), process);
return res;
}
public boolean hasProcessForPackage(DVMPackage package_in) {
for (MachineProcess process : Data.values()) {
if (
process.machineAddress.equals(package_in.machine_address) &&
process.machinePort == package_in.machine_port &&
process.userName.equals(package_in.user_name) &&
process.userWorkspace.equals(package_in.user_workspace)
) {
return true;
}
}
return false;
}
}

27
src/TestingSystem/Common/TestingPlanner.java
View File

@@ -21,6 +21,7 @@ public abstract class TestingPlanner<P extends TestingPackage> {
}
//---
protected Object ServerCommand(ServerCode code_in, String arg, Serializable object_in) throws Exception {
System.out.println("Команда серверу " + code_in.toString() + "arg=" + arg + " object=" + object_in);
TestingSystemPass<Object> pass = new TestingSystemPass<Object>() {
@Override
public String getDescription() {
@@ -31,10 +32,13 @@ public abstract class TestingPlanner<P extends TestingPackage> {
Command(new ServerExchangeUnit_2021(code_in, arg, object_in));
target = response.object;
}
@Override
protected boolean validate() {
return Log.isEmpty();
}
};
if (!pass.Do()) {
ServerConnectionError(code_in);
}
if (!pass.Do())
ServerConnectionError(code_in, pass.Log.toString());
return pass.target;
}
protected Object ServerCommand(ServerCode code_in, Serializable object_in) throws Exception {
@@ -105,22 +109,22 @@ public abstract class TestingPlanner<P extends TestingPackage> {
}
protected void Disconnect() {
}
protected void MachineConnectionError(){
protected void MachineConnectionError() {
}
protected void ServerConnectionError(ServerCode code_in) throws Exception{
protected void ServerConnectionError(ServerCode code_in, String logText) throws Exception {
throw new PassException("Ошибка взаимодействия с сервером " + code_in);
}
// ---
protected void PerformPackage(TestingPackage package_in) throws Exception{
protected void PerformPackage(TestingPackage package_in) throws Exception {
testingPackage = (P) package_in;
//--
Print(testingPackage.id + ":" + testingPackage.state.getDescription());
//--
if (testingPackage.connectionErrosCount>=10){
Print(testingPackage.id+" had 10 connection errors. stop");
if (testingPackage.connectionErrosCount >= 10) {
Print(testingPackage.id + " had 10 connection errors. stop");
UpdatePackageState(TasksPackageState.ConnectionError);
MachineConnectionError();
}else {
} else {
//--
InitSessionCredentials();
if (testingPackage.state.equals(TasksPackageState.Analysis)) {
@@ -182,14 +186,13 @@ public abstract class TestingPlanner<P extends TestingPackage> {
testingPackage = null;
System.gc();
//--
}
public void Perform() {
try {
testingPackage = null;
Vector<P> activePackages = (Vector<P>) ServerCommand(getActivePackagesCode());
// System.out.println(this.getClass().getSimpleName()+": found "+activePackages.size()+" active packages"); //Тесты:
for (P activePackage: activePackages)
// System.out.println(this.getClass().getSimpleName()+": found "+activePackages.size()+" active packages"); //Тесты:
for (P activePackage : activePackages)
PerformPackage(activePackage);
} catch (Exception ex) {
ex.printStackTrace();

49
src/TestingSystem/Common/TestingServer.java
View File

@@ -12,13 +12,11 @@ import Repository.Server.ServerCode;
import Repository.Server.ServerExchangeUnit_2021;
import TestingSystem.Common.Group.Group;
import TestingSystem.Common.MachineProcess.MachineProcess;
import TestingSystem.Common.MachineProcess.MachineProcessState;
import TestingSystem.Common.Test.Test;
import TestingSystem.Common.Test.TestType;
import TestingSystem.Common.TestingPackageToKill.TestingPackageToKill;
import TestingSystem.DVM.DVMPackage.DVMPackage;
import TestingSystem.DVM.DVMPackage.DVMPackage_json;
import TestingSystem.DVM.DVMTestingPlanner;
import TestingSystem.SAPFOR.Json.SapforPackage_json;
import TestingSystem.SAPFOR.SapforConfiguration.SapforConfiguration;
import TestingSystem.SAPFOR.SapforConfigurationCommand.SapforConfigurationCommand;
@@ -120,65 +118,56 @@ public class TestingServer extends RepositoryServer<TestsDatabase> {
protected void startAdditionalThreads() {
testingThread.start();
}
protected DVMTestingPlanner DVMTestingPlanner = new DVMTestingPlanner();
protected SapforTestingPlanner sapforTestingPlanner = new SapforTestingPlanner();
//--
void startNecessaryMachines() {
System.out.println("Запуск необходимых нитей машин...");
//todo. когда БД машин будет перенесена только на сервер. просто идти по всем машинам.
System.out.println("Проверка процессов машин...");
try {
LinkedHashMap<String, MachineProcess> active_processes = db.machinesProcesses.getSortedByKeys();
Vector<String> aborted = new Vector<>();
for (MachineProcess process : db.machinesProcesses.Data.values()) {
if (process.isAborted()) {
aborted.add(process.id);
}
}
//---
for (String key : aborted) {
System.out.println(key + " остановлен");
db.machinesProcesses.Data.remove(key);
}
//---
System.out.println(db.machinesProcesses.size() + " активных процессов");
//--
LinkedHashMap<String, MachineProcess> processes_to_start = new LinkedHashMap<>();
//1. Получить список всех пакетов, которые активны, и взять из них машины.
for (DVMPackage dvmPackage : db.dvmPackages.Data.values()) {
if (dvmPackage.state.isActive()) {
//-
Vector<String> k = new Vector<>();
k.add(dvmPackage.machine_address);
k.add(String.valueOf(dvmPackage.machine_port));
k.add(dvmPackage.user_name);
k.add(dvmPackage.user_workspace);
//-
String key = String.join("_", k);
if (!active_processes.containsKey(key)) {
if (!db.machinesProcesses.hasProcessForPackage(dvmPackage)) {
MachineProcess new_process = new MachineProcess(dvmPackage);
processes_to_start.put(key, new_process);
processes_to_start.put(new_process.getUniqueKey(), new_process);
}
}
}
System.out.println("Активные процессы: " + active_processes.size());
for (String key : active_processes.keySet()) {
System.out.println(key);
}
System.out.println("-------------------");
System.out.println("Ожидающие запуск процессы: " + processes_to_start.size());
for (String key : processes_to_start.keySet()) {
System.out.println(key);
}
System.out.println("-------------------");
//запуск.
for (MachineProcess process : processes_to_start.values()) {
System.out.println("Запуск процесса " + process.getUniqueKey() + " ...");
process.Start();
if (Utils.checkFileCreation(process.getStartedFile())) {
process.state = MachineProcessState.Active;
db.machinesProcesses.Data.put(process.id, process);
System.out.println("Выполнено");
} else {
System.out.println("Не удалось запустить процесс.");
System.out.println(process.getUniqueKey()+" запущен");
}
}
} catch (Exception ex) {
ex.printStackTrace();
}
System.out.println("done");
}
//--
protected Thread testingThread = new Thread(() -> {
while (true) {
// DVMTestingPlanner.Perform();
// sapforTestingPlanner.Perform();
startNecessaryMachines();
sapforTestingPlanner.Perform();
System.out.println("sleep...");
Utils.sleep(5000);
}

385
src/TestingSystem/DVM/DVMTestingPlanner.java
View File

@@ -1,385 +0,0 @@
package TestingSystem.DVM;
import Common.Constants;
import Common.Global;
import Common.Utils.Utils;
import GlobalData.Machine.Machine;
import GlobalData.RemoteFile.RemoteFile;
import GlobalData.Tasks.TaskState;
import GlobalData.User.User;
import ProjectData.Files.ProjectFile;
import ProjectData.LanguageName;
import Repository.Server.ServerCode;
import TestingSystem.Common.TasksPackageState;
import TestingSystem.Common.TestingPlanner;
import TestingSystem.DVM.DVMPackage.DVMPackage;
import TestingSystem.DVM.DVMTasks.DVMCompilationTask;
import TestingSystem.DVM.DVMTasks.DVMRunTask;
import TestingSystem.DVM.DVMTasks.DVMTask;
import Visual_DVM_2021.Passes.All.UnzipFolderPass;
import javafx.util.Pair;
import org.apache.commons.io.FileUtils;
import java.io.File;
import java.io.FileFilter;
import java.nio.charset.Charset;
import java.nio.file.Paths;
import java.util.*;
public class DVMTestingPlanner extends TestingPlanner<DVMPackage> {
//----
LinkedHashMap<String, Machine> machines = new LinkedHashMap<>();
LinkedHashMap<String, User> users = new LinkedHashMap<>();
//-- текущие машина и пользователь.
Machine machine = null;
User user = null;
//----
protected RemoteFile packageRemoteWorkspace;
protected File packageLocalWorkspace;
//--
public String getPlanner() {
return String.join("/", user.workspace, "modules", "planner");
}
//--
//Получить ид тестов и их папки на сервере.
LinkedHashMap<Integer, File> getTestsFromJson() {
LinkedHashMap<Integer, File> res = new LinkedHashMap<>();
for (DVMCompilationTask task : testingPackage.package_json.compilationTasks) {
if (!res.containsKey(task.test_id)) {
res.put(task.test_id, Paths.get(Global.TestsDirectory.getAbsolutePath(), String.valueOf(task.test_id)).toFile());
}
}
return res;
}
static LinkedHashMap<LanguageName, Vector<ProjectFile>> getTestPrograms(File test) {
LinkedHashMap<LanguageName, Vector<ProjectFile>> res = new LinkedHashMap<>();
//--
res.put(LanguageName.fortran, new Vector<>());
res.put(LanguageName.c, new Vector<>());
res.put(LanguageName.cpp, new Vector<>());
//--
File[] files = test.listFiles(new FileFilter() {
@Override
public boolean accept(File pathname) {
return pathname.isFile();
}
});
if (files != null) {
for (File file : files) {
ProjectFile projectFile = new ProjectFile(new File(file.getName()));
if (projectFile.isNotExcludedProgram()) res.get(projectFile.languageName).add(projectFile);
}
}
return res;
}
static void generateForLanguage(String dvm_drv, LanguageName language, Vector<ProjectFile> language_programs, Vector<String> titles, Vector<String> objects, Vector<String> bodies, String flags) {
if (!language_programs.isEmpty()) {
String LANG_ = language.toString().toUpperCase() + "_";
Vector<String> module_objects = new Vector<>();
String module_body = "";
int i = 1;
for (ProjectFile program : language_programs) {
//--
String object = Utils.DQuotes(language + "_" + i + ".o");
module_objects.add(object);
module_body += object + ":\n" + "\t" + String.join(" ", Utils.MFVar(LANG_ + "COMMAND"), Utils.MFVar(LANG_ + "FLAGS"), program.getStyleOptions(), "-c", program.getQSourceName(), "-o", object + "\n\n");
++i;
}
titles.add(String.join("\n", LANG_ + "COMMAND=" + Utils.DQuotes(dvm_drv) + " " + language.getDVMCompile(), LANG_ + "FLAGS=" + flags, LANG_ + "OBJECTS=" + String.join(" ", module_objects), ""));
objects.add(Utils.MFVar(LANG_ + "OBJECTS"));
bodies.add(module_body);
}
}
static String generateMakefile(File test, LanguageName test_language, String dvm_drv, String flags) {
//----->>
LinkedHashMap<LanguageName, Vector<ProjectFile>> programs = getTestPrograms(test);
Vector<String> titles = new Vector<>();
Vector<String> objects = new Vector<>();
Vector<String> bodies = new Vector<>();
String binary = Utils.DQuotes("0");
//----->>
for (LanguageName languageName : programs.keySet()) {
generateForLanguage(dvm_drv, languageName, programs.get(languageName), titles, objects, bodies, flags);
}
//----->>
return String.join("\n", "LINK_COMMAND=" + Utils.DQuotes(dvm_drv) + " " + test_language.getDVMLink(), "LINK_FLAGS=" + flags + "\n", String.join("\n", titles), "all: " + binary, binary + " : " + String.join(" ", objects), "\t" + Utils.MFVar("LINK_COMMAND") + " " + Utils.MFVar("LINK_FLAGS") + " " + String.join(" ", objects) + " -o " + binary, String.join(" ", bodies));
}
public void getTasksInfo(List<? extends DVMTask> tasks, String file_name) throws Exception {
LinkedHashMap<Integer, DVMTask> sorted_tasks = new LinkedHashMap<>();
for (DVMTask task : tasks)
sorted_tasks.put(task.id, task);
//--
File info_file = Paths.get(packageLocalWorkspace.getAbsolutePath(), "results", file_name).toFile();
List<String> lines = FileUtils.readLines(info_file, Charset.defaultCharset());
for (String packed : lines) {
if (!packed.isEmpty()) {
String[] data = packed.split(" ");
int id = Integer.parseInt(data[0]);
TaskState state = TaskState.valueOf(data[1]);
double time = Double.parseDouble(data[2]);
//--
DVMTask task = sorted_tasks.get(id);
task.state = state;
task.Time = state.equals(TaskState.AbortedByTimeout) ? (task.maxtime + 1) : time;
}
}
}
//---
@Override
protected ServerCode getActivePackagesCode() {
return ServerCode.GetFirstsActiveDVMPackages;
}
@Override
protected ServerCode getCheckIfNeedsKillCode() {
return ServerCode.DVMPackageNeedsKill;
}
@Override
protected TasksPackageState getStateAfterStart() {
return TasksPackageState.CompilationWorkspacesCreation;
}
@Override
protected void TestsSynchronize() throws Exception {
testingPackage.readJson();
LinkedHashMap<Integer, File> tests = getTestsFromJson();
//синхронизировать их.
for (int test_id : tests.keySet()) {
// Print("testId="+test_id);
File test = tests.get(test_id);
RemoteFile test_dst = new RemoteFile(testingPackage.user_workspace + "/projects/" + test_id, true);
// Print("src="+test.getAbsolutePath());
// Print("dst="+test_dst.full_name);
user.connection.MKDIR(test_dst);
user.connection.SynchronizeSubDirsR(test, test_dst);
// Print("done");
}
}
@Override
protected void PackageWorkspaceCreation() throws Exception {
if (!CheckModules()) {
return;
}
//--
testingPackage.readJson();
//--
LinkedHashMap<Integer, File> tests = getTestsFromJson();
//создать папку для пакета.
user.connection.sftpChannel.mkdir(packageRemoteWorkspace.full_name);
//положить туда запакованные тексты задач.
Vector<String> compilationLines = new Vector<>();
Vector<String> runLines = new Vector<>();
for (DVMCompilationTask compilationTask : testingPackage.package_json.compilationTasks) {
String makefileText = generateMakefile(tests.get(compilationTask.test_id), compilationTask.language, testingPackage.drv, compilationTask.flags);
compilationLines.addAll(compilationTask.pack(makefileText));
for (DVMRunTask runTask : compilationTask.runTasks)
runLines.addAll(runTask.pack(null));
}
RemoteFile compilationPackage = new RemoteFile(packageRemoteWorkspace, "compilationTasks");
RemoteFile runPackage = new RemoteFile(packageRemoteWorkspace, "runTasks");
user.connection.writeToFile(String.join("\n", compilationLines) + "\n", compilationPackage);
user.connection.writeToFile(String.join("\n", runLines) + "\n", runPackage);
// --
user.connection.MKDIR(new RemoteFile(packageRemoteWorkspace, "state"));
}
@Override
protected void PackageStart() throws Exception {
String plannerStartCommand = String.join(" ",
Utils.DQuotes(getPlanner()),
Utils.DQuotes(user.workspace),
Utils.DQuotes(packageRemoteWorkspace.full_name),
Utils.DQuotes(testingPackage.kernels),
Utils.DQuotes(testingPackage.drv));
testingPackage.PID = user.connection.startShellProcess(packageRemoteWorkspace,"PID",
"ulimit -s unlimited", plannerStartCommand);
System.out.println("PID="+Utils.Brackets(testingPackage.PID));
RemoteFile STARTED = new RemoteFile(packageRemoteWorkspace, "STARTED");
while (!user.connection.Exists(STARTED)) {
Print("waiting for package start...");
Utils.sleep(1000);
}
}
@Override
protected boolean CheckNextState() throws Exception {
boolean progress_changed = false;
boolean state_changed = false;
RemoteFile progress = new RemoteFile(packageRemoteWorkspace, "progress");
if (user.connection.Exists(progress)) {
String s = user.connection.readFromFile(progress);
int current_progress = Integer.parseInt(s);
if (current_progress != testingPackage.progress) {
Print("progress changed: " + current_progress);
testingPackage.progress = current_progress;
progress_changed = true;
}
}
RemoteFile stateDir = new RemoteFile(packageRemoteWorkspace, "state");
//состояния пакета могут меняться только по возрастанию. ищем, появилось ли такое.
Vector<TasksPackageState> higherStates = testingPackage.state.getHigherStates();
Collections.reverse(higherStates); //берем в обратном порядке, чтобы быстрее найти высшее.
for (TasksPackageState state : higherStates) {
RemoteFile file = new RemoteFile(stateDir, state.toString());
if (user.connection.Exists(file)) {
Print("found new state: " + file.name);
testingPackage.state = state;
state_changed = true;
break;
}
}
//--
user.connection.iterations++;
if (user.connection.iterations == 100) {
Disconnect();
}
//--
return progress_changed || state_changed;
}
@Override
protected void DownloadResults() throws Exception {
Utils.CheckDirectory(packageLocalWorkspace);
RemoteFile remote_results_archive = new RemoteFile(packageRemoteWorkspace, "results.zip");
File results_archive = new File(packageLocalWorkspace, "results.zip");
user.connection.performScript(packageRemoteWorkspace, "zip -r " + Utils.DQuotes("results.zip") + " " + Utils.DQuotes("results"));
//---
if (user.connection.Exists(remote_results_archive)) {
user.connection.getSingleFile(remote_results_archive.full_name, results_archive.getAbsolutePath());
UnzipFolderPass unzipFolderPass = new UnzipFolderPass();
unzipFolderPass.Do(results_archive.getAbsolutePath(), packageLocalWorkspace.getAbsolutePath(), false);
}
//---
if (Global.properties.eraseTestingWorkspaces && user.connection.Exists(packageRemoteWorkspace))
user.connection.RMDIR(packageRemoteWorkspace.full_name);
}
@Override
protected void AnalyseResults() throws Exception {
testingPackage.readJson();
Print("analysing results");
Vector<DVMRunTask> runTasks = new Vector<>();
for (DVMCompilationTask compilationTask : testingPackage.package_json.compilationTasks)
runTasks.addAll(compilationTask.runTasks);
//----
getTasksInfo(testingPackage.package_json.compilationTasks, "CompilationInfo.txt");
getTasksInfo(runTasks, "RunningInfo.txt");
//--
int ct_count = 0;
int rt_count = 0;
//--
for (DVMCompilationTask compilationTask : testingPackage.package_json.compilationTasks) {
compilationTask.dvm_package_id = testingPackage.id;
ct_count++;
File ct_workspace = Paths.get(packageLocalWorkspace.getAbsolutePath(), "results", String.valueOf(compilationTask.id)).toFile();
if (ct_workspace.exists()) {
for (DVMRunTask runTask : compilationTask.runTasks) {
runTask.dvm_package_id = testingPackage.id;
rt_count++;
runTask.compilation_state = compilationTask.state;
runTask.compilation_time = compilationTask.Time;
if (compilationTask.state == TaskState.DoneWithErrors) {
runTask.state = TaskState.Canceled;
} else {
File rt_workspace = Paths.get(packageLocalWorkspace.getAbsolutePath(), "results", String.valueOf(runTask.id)).toFile();
if (rt_workspace.exists() && runTask.state.equals(TaskState.Finished)) {
//анализ задачи на запуск.
File outFile = new File(rt_workspace, Constants.out_file);
File errFile = new File(rt_workspace.getAbsolutePath(), Constants.err_file);
//--
String output = FileUtils.readFileToString(outFile);
String errors = FileUtils.readFileToString(errFile);
//--
List<String> output_lines = Arrays.asList(output.split("\n"));
List<String> errors_lines = Arrays.asList(errors.split("\n"));
//---
if (Utils.isCrushed(output_lines, errors_lines)) {
runTask.state = TaskState.Crushed;
} else {
Pair<TaskState, Integer> results = new Pair<>(TaskState.Done, 100);
switch (runTask.test_type) {
case Correctness:
results = Utils.analyzeCorrectness(output_lines);
break;
case Performance:
results = Utils.analyzePerformance(output_lines);
break;
default:
break;
}
runTask.state = results.getKey();
runTask.progress = results.getValue();
runTask.CleanTime = Utils.parseCleanTime(output);
}
}
}
}
}
}
testingPackage.progress = 100;
testingPackage.saveJson(); //запись обновленных результатов пакета в json!
Print("analysis done, ct_count=" + ct_count + " rt count=" + rt_count);
}
@Override
protected void Kill() throws Exception {
if (!testingPackage.PID.isEmpty()) {
user.connection.Command("kill -9 " + testingPackage.PID);
}
}
@Override
protected void InitSessionCredentials() {
String machine_url = testingPackage.machine_address + ":" + testingPackage.machine_port;
if (!machines.containsKey(machine_url)) machines.put(machine_url, testingPackage.getMachine());
if (!users.containsKey(testingPackage.user_name)) users.put(testingPackage.user_name, testingPackage.getUser());
//-->>
machine = machines.get(machine_url);
user = users.get(testingPackage.user_name);
//--
packageRemoteWorkspace = new RemoteFile(user.workspace + "/tests", String.valueOf(testingPackage.id), true);
packageLocalWorkspace = new File(Global.DVMPackagesDirectory, String.valueOf(testingPackage.id));
}
@Override
protected boolean Connect() {
if (user.connection == null) {
try {
user.connection = new UserConnection(machine, user);
// Print("Соединение c " + machine.getURL() + " " + user.login + " успешно установлено.");
} catch (Exception ex) {
Print(ex.toString());
user.connection = null;
Print("Не удалось установить соединение.");
}
}
return user.connection != null;
}
@Override
protected void Disconnect() {
if (user.connection != null) {
user.connection.Disconnect();
user.connection = null;
}
}
//todo потом обобщить и для одиночных запусков. если понадобится.
protected boolean CheckModules() throws Exception {
RemoteFile modulesDirectory = new RemoteFile(user.workspace, "modules");
RemoteFile version = new RemoteFile(modulesDirectory, "version.h");
int current_version = Constants.Nan;
int actual_version = Constants.planner_version;
if (user.connection.Exists(version)) {
try {
current_version = Integer.parseInt(user.connection.readFromFile(version));
} catch (Exception ex) {
ex.printStackTrace();
}
}
if (current_version < actual_version) {
Print("Закачка кода модулей...");
for (String resource_name : Constants.resourses_names) {
Print(resource_name);
user.connection.putResource(modulesDirectory, resource_name);
}
//--
Print("Сборка модулей...");
String modules_log = user.connection.compileModules(modulesDirectory);
if (!modules_log.isEmpty()) {
testingPackage.description = modules_log;
testingPackage.state = TasksPackageState.Aborted;
return false;
}
}
return true;
}
}

60
src/TestingSystem/DVM/MachineQueueSupervisor.java
View File

@@ -129,6 +129,7 @@ public class MachineQueueSupervisor extends TestingPlanner<DVMPackage> {
protected void Print(String message) {
try {
if (isPrintOn()) {
// System.out.println(message);
Global.Log.Print(message);
}
} catch (Exception ex) {
@@ -136,9 +137,10 @@ public class MachineQueueSupervisor extends TestingPlanner<DVMPackage> {
}
}
void Finalize(String reason) {
Print(reason);
File stateFile = new File(supervisorHome, Constants.ABORTED);
try {
FileUtils.writeStringToFile(stateFile, reason);
FileUtils.writeStringToFile(stateFile,reason);
} catch (Exception ex) {
ex.printStackTrace();
}
@@ -457,14 +459,11 @@ public class MachineQueueSupervisor extends TestingPlanner<DVMPackage> {
}
@Override
protected void MachineConnectionError() {
System.out.println("MACHINE CONNECTION ERROR");
Finalize("Количество безуспешных попыток соединения с машиной " + machine.getURL() +
" превысило 10");
Finalize("Количество безуспешных попыток соединения с машиной " + machine.getURL() + " превысило 10");
}
@Override
protected void ServerConnectionError(ServerCode code_in) throws Exception {
System.out.println("NO SERVER");
Finalize("Не удалось выполнить команду " + code_in + " на сервере тестирования");
protected void ServerConnectionError(ServerCode code_in, String logText) throws Exception {
Finalize("Не удалось выполнить команду " + code_in + " на сервере тестирования\n" + logText);
}
@Override
protected void Kill() throws Exception {
@@ -473,28 +472,35 @@ public class MachineQueueSupervisor extends TestingPlanner<DVMPackage> {
}
}
@Override
public void Perform() {
try {
String currentServerName = (String) ServerCommand(ServerCode.GetServerName);
System.out.println("current serverName="+Utils.Brackets(currentServerName));
System.out.println("serverName="+Utils.Brackets(serverName));
if (!serverName.equals(currentServerName)){
Finalize("Устаревшее имя сервера");
}
testingPackage = null;
Vector<DVMPackage> activePackages = (Vector<DVMPackage>) ServerCommand(getActivePackagesCode(), machine.getURL(), null);
// System.out.println(this.getClass().getSimpleName()+": found "+activePackages.size()+" active packages");
for (DVMPackage activePackage : activePackages)
PerformPackage(activePackage);
} catch (Exception ex) {
ex.printStackTrace();
} finally {
Utils.sleep(getSleepMillis());
}
}
@Override
protected void InitSessionCredentials() {
packageRemoteWorkspace = new RemoteFile(user.workspace + "/tests", String.valueOf(testingPackage.id), true);
packageLocalWorkspace = new File(Global.DVMPackagesDirectory, String.valueOf(testingPackage.id));
}
@Override
public void Perform() {
try {
Print("Проверка сервера...");
String currentServerName = (String) ServerCommand(ServerCode.GetServerName);
Print("имя текущего сервера " + Utils.Brackets(currentServerName));
Print("имя сервера, создавшего нить " + Utils.Brackets(serverName));
if (!serverName.equals(currentServerName)){
Finalize("Несоответствующий сервер");
}
Print("Запрос активных пакетов для машины "+Utils.Brackets(machine.getURL()));
testingPackage = null;
Vector<DVMPackage> activePackages = (Vector<DVMPackage>) ServerCommand(getActivePackagesCode(), machine.getURL(), null);
if (activePackages.isEmpty())
Finalize("Не найдено активных пакетов для машины "+Utils.Brackets(machine.getURL()));
for (DVMPackage activePackage : activePackages)
PerformPackage(activePackage);
} catch (Exception ex) {
Print("Exception occured");
Print(ex.getMessage());
ex.printStackTrace();
} finally {
Print("sleep");
Utils.sleep(getSleepMillis());
}
}
}

2
src/TestingSystem/SAPFOR/SapforTestingPlanner.java
View File

@@ -61,7 +61,7 @@ public class SapforTestingPlanner extends TestingPlanner<SapforPackage> {
File visualiser = new File(workspace, "VisualSapfor.jar");
FileUtils.copyFile(new File(Global.Home, "TestingSystem.jar"), visualiser);
//создание настроек
GlobalProperties properties = new GlobalProperties();
GlobalProperties properties = new GlobalProperties(Global.properties);
properties.Mode = Current.Mode.Package;
Utils.jsonToFile(properties, new File(workspace, "properties"));
//подготовка пакетного режима. Запустит его уже очередь.

2
src/Visual_DVM_2021/Passes/Pass_2021.java
View File

@@ -30,7 +30,7 @@ public class Pass_2021<T> {
public Semaphore animation_sem;
//--------------
public Vector<PassControl> controls = new Vector<>();
protected TextLog Log; //внутренний журнал прохода.
public TextLog Log; //внутренний журнал прохода.
protected boolean interrupt;
protected SwingWorker dispatcher = null;
private PassForm form = null;