M/VisualSapfor
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

рефакторинг нити планировщика двм тестов на сервере.

Browse Source
This commit is contained in:
02090095
2023-12-14 18:45:41 +03:00
parent c07b72eb5d
commit 1fc60e6c12
11 changed files with 403 additions and 177 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
.idea/workspace.xml generated
View File

@@ -7,24 +7,17 @@
</component>
<component name="ChangeListManager">
<list default="true" id="e42177c3-2328-4b27-8a01-35779b2beb99" name="Default Changelist" comment="">
<change afterPath="$PROJECT_DIR$/src/TestingSystem/Common/TestingPlanner_OLD.java" afterDir="false" />
<change afterPath="$PROJECT_DIR$/src/TestingSystem/DVM/DVMTestingPlanner.java" afterDir="false" />
<change afterPath="$PROJECT_DIR$/src/Visual_DVM_2021/Passes/All/DeleteDVMPackage.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/Packages/dvmpackage_1702386255/package_json" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/src/Common/UI/Menus_2023/DVMPackagesBar/DVMPackagesBar.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/Common/UI/Menus_2023/DVMPackagesBar/DVMPackagesBar.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/ProjectData/Files/ProjectFile.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/ProjectData/Files/ProjectFile.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/Repository/Server/ServerCode.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/Repository/Server/ServerCode.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/TestingSystem/Common/TestingPackage/TestingPackage.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/TestingSystem/Common/TestingPackage/TestingPackage.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/TestingSystem/Common/TestingPlanner.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/TestingSystem/Common/TestingPlanner.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/TestingSystem/Common/TestingServer.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/TestingSystem/Common/TestingServer.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/TestingSystem/Common/TestsDatabase.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/TestingSystem/Common/TestsDatabase.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/TestingSystem/DVM/DVMPackage/DVMPackage.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/TestingSystem/DVM/DVMPackage/DVMPackage.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/TestingSystem/DVM/DVMPackage/DVMPackageDBTable.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/TestingSystem/DVM/DVMPackage/DVMPackageDBTable.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/TestingSystem/DVM/TestsSupervisor_2022.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/TestingSystem/DVM/TestsSupervisor_2022.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/TestingSystem/SAPFOR/SapforTasksPackageSupervisor/SapforTasksPackageSupervisor.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/TestingSystem/SAPFOR/SapforTasksPackageSupervisor/SapforTasksPackageSupervisor.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/Visual_DVM_2021/Passes/All/AddDVMPackage.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/Visual_DVM_2021/Passes/All/AddDVMPackage.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/Visual_DVM_2021/Passes/All/AddTasksToDVMPackage.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/Visual_DVM_2021/Passes/All/AddTasksToDVMPackage.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/Visual_DVM_2021/Passes/PassCode_2021.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/Visual_DVM_2021/Passes/PassCode_2021.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/TestingSystem/DVM/DVMTasks/DVMCompilationTask.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/TestingSystem/DVM/DVMTasks/DVMCompilationTask.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/TestingSystem/DVM/DVMTasks/DVMRunTask.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/TestingSystem/DVM/DVMTasks/DVMRunTask.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/TestingSystem/DVM/DVMTasks/DVMTask.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/TestingSystem/DVM/DVMTasks/DVMTask.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/TestingSystem/DVM/DVMTestingPlanner.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/TestingSystem/DVM/DVMTestingPlanner.java" afterDir="false" />
</list>
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />

7
src/ProjectData/Files/ProjectFile.java
View File

@@ -16,7 +16,7 @@ public class ProjectFile extends DBObject {
public FileState state = FileState.Undefined; //состояние файла.
public ProjectFile() {
}
public ProjectFile(File file_in){
public ProjectFile(File file_in) {
file = file_in;
AutoDetectProperties();
}
@@ -24,7 +24,7 @@ public class ProjectFile extends DBObject {
public Object getPK() {
return file.getName();
}
public void AutoDetectProperties(){
public void AutoDetectProperties() {
AutoDetectProperties(file.getName());
}
public void AutoDetectProperties(String name_in) {
@@ -153,4 +153,7 @@ public class ProjectFile extends DBObject {
}
return "";
}
public boolean isNotExcludedProgram() {
return fileType.equals(FileType.program) && !languageName.equals(LanguageName.n) && !state.equals(FileState.Excluded);
}
}

5
src/Repository/Server/ServerCode.java
View File

@@ -86,5 +86,8 @@ public enum ServerCode {
ActualizeSAPFORPackages,
ActualizeDVMPackages,
// PublishTestProject
GetFirstActiveDVMPackage;
GetFirstActiveDVMPackage,
DVMPackageNeedsKill
;
}

1
src/TestingSystem/Common/TestingPackage/TestingPackage.java
View File

@@ -29,4 +29,5 @@ public class TestingPackage extends riDBObject {
progress = tasksPackage.progress;
state = tasksPackage.state;
}
public void destructor(){}
}

19
src/TestingSystem/Common/TestingPlanner.java
View File

@@ -78,13 +78,13 @@ public abstract class TestingPlanner<P extends TestingPackage> {
protected abstract TasksPackageState getStateAfterStart();
protected void InitSessionCredentials() {
}
protected abstract void TestsSynchronize();
protected abstract void PackageWorkspaceCreation();
protected abstract void AnalyseResults();
protected abstract void PackageStart();
protected abstract boolean CheckNextState();
protected abstract void DownloadResults();
protected abstract void Kill();
protected abstract void TestsSynchronize() throws Exception;
protected abstract void PackageWorkspaceCreation() throws Exception;
protected abstract void AnalyseResults() throws Exception;
protected abstract void PackageStart() throws Exception;
protected abstract boolean CheckNextState() throws Exception;
protected abstract void DownloadResults() throws Exception;
protected abstract void Kill() throws Exception;
protected boolean Connect() {
return true;
}
@@ -152,6 +152,11 @@ public abstract class TestingPlanner<P extends TestingPackage> {
Disconnect();
}
}
//--
testingPackage.destructor();
testingPackage = null;
System.gc();
//--
}
} catch (Exception ex) {
ex.printStackTrace();

21
src/TestingSystem/Common/TestingServer.java
View File

@@ -379,10 +379,29 @@ public class TestingServer extends RepositoryServer<TestsDatabase> {
response = new ServerExchangeUnit_2021(ServerCode.OK);
response.object = db.getFirstActiveDVMPackage();
break;
case DVMPackageNeedsKill:
DVMPackageNeedsKill();
break;
default:
throw new RepositoryRefuseException("Неподдерживаемый код: " + code);
}
}
private void DVMPackageNeedsKill() {
response = new ServerExchangeUnit_2021(ServerCode.OK);
response.object = false;
/*
int packageId = (int) request.object;
boolean res_ = false;
for (TasksPackageToKill tasksPackageToKill : db.packagesToKill.Data.values()) {
if (tasksPackageToKill.packageId == packageId) {
res_ = true;
break;
}
}
response.object = res_;
*/
}
public void UpdateTestTasks() throws Exception {
Vector<TestCompilationTask> compilationTasks = (Vector<TestCompilationTask>) request.object;
account_db.BeginTransaction();
@@ -596,7 +615,7 @@ public class TestingServer extends RepositoryServer<TestsDatabase> {
+ "\nТест будет удален"
);
}
}else if (object instanceof DVMPackage) {
} else if (object instanceof DVMPackage) {
DVMPackage dvmPackage = (DVMPackage) object;
//--
dvmPackage.state = TasksPackageState.Queued;

16
src/TestingSystem/DVM/DVMPackage/DVMPackage.java
View File

@@ -2,6 +2,9 @@ package TestingSystem.DVM.DVMPackage;
import Common.Database.DBObject;
import Common.Global;
import Common.Utils.Utils;
import GlobalData.Machine.Machine;
import GlobalData.Machine.MachineType;
import GlobalData.User.User;
import TestingSystem.Common.TestingPackage.TestingPackage;
import com.sun.org.glassfish.gmbal.Description;
@@ -21,6 +24,10 @@ public class DVMPackage extends TestingPackage {
public DVMPackage_json package_json = null;
//---
@Override
public void destructor() {
package_json = null;
}
@Override
public void SynchronizeFields(DBObject src) {
super.SynchronizeFields(src);
DVMPackage tasksPackage = (DVMPackage) src;
@@ -41,4 +48,13 @@ public class DVMPackage extends TestingPackage {
public void saveJson() throws Exception {
Utils.jsonToFile(package_json, getJsonFile());
}
public void readJson() throws Exception {
package_json = (DVMPackage_json) Utils.jsonFromFile(getJsonFile(), DVMPackage_json.class);
}
public Machine getMachine(){
return new Machine(machine_name,machine_address,machine_port,MachineType.Server);
}
public User getUser(){
return new User(user_name, user_password, user_workspace);
}
}

11
src/TestingSystem/DVM/DVMTasks/DVMCompilationTask.java
View File

@@ -19,7 +19,16 @@ public class DVMCompilationTask extends DVMTask {
String flags_in
) {
super(configuration, group, test, flags_in);
flags = flags_in;
maxtime = configuration.c_maxtime;
}
@Override
public Vector<String> pack(Object arg) {
Vector<String> res = new Vector<>();
res.add(String.valueOf(id)); //1
res.add(String.valueOf(maxtime)); //2
res.add(String.valueOf(test_id)); //3
res.add(arg.toString().replace("\n", "|")); //4
//игнор аргумента. ядро всегда одно.
return res;
}
}

7
src/TestingSystem/DVM/DVMTasks/DVMRunTask.java
View File

@@ -29,8 +29,6 @@ public class DVMRunTask extends DVMTask {
@Expose
public int progress = 0;
@Expose
public LanguageName language = LanguageName.fortran;
@Expose
public int cube = 1;
@Expose
public int min_dim = 1;
@@ -56,7 +54,6 @@ public class DVMRunTask extends DVMTask {
super(configuration, group, test, flags_in);
//--------------------------
//инфа о компиляции.
language = group.language;
compilation_maxtime = configuration.c_maxtime;
compilation_state = TaskState.Waiting;
//инфа о запуске
@@ -99,7 +96,7 @@ public class DVMRunTask extends DVMTask {
}
//-
@Override
public Vector<String> pack(int kernels_in) {
public Vector<String> pack(Object arg) {
Vector<String> res = new Vector<>();
res.add(String.valueOf(id)); //1
res.add(String.valueOf(maxtime)); //2
@@ -108,7 +105,7 @@ public class DVMRunTask extends DVMTask {
res.add(environments); //5
res.add(usr_par.replace("\n", "|")); //6
res.add(args); //7
res.add(String.valueOf(kernels_in)); //8
res.add(String.valueOf(kernels)); //8
return res;
}
public String getEnvironments() {

7
src/TestingSystem/DVM/DVMTasks/DVMTask.java
View File

@@ -3,6 +3,7 @@ import Common.Constants;
import Common.Database.DBObject;
import Common.Database.iDBObject;
import GlobalData.Tasks.TaskState;
import ProjectData.LanguageName;
import TestingSystem.Common.Group.Group;
import TestingSystem.Common.Test.Test;
import TestingSystem.Common.Test.TestType;
@@ -22,6 +23,8 @@ public class DVMTask extends iDBObject {
@Expose
public String test_description = "";
@Expose
public LanguageName language = LanguageName.fortran;
@Expose
public String flags = "";
@Expose
public int kernels = 1;
@@ -43,6 +46,7 @@ public class DVMTask extends iDBObject {
group_description = t.group_description;
test_id = t.test_id;
test_description = t.test_description;
language = t.language;
flags = t.flags;
kernels = t.kernels;
state = t.state;
@@ -62,9 +66,10 @@ public class DVMTask extends iDBObject {
group_description = group.description;
test_description = test.description;
test_type = group.type;
language = group.language;
flags = flags_in;
}
public Vector<String> pack(int kernels) {
public Vector<String> pack(Object arg) {
return null;
}
}

465
src/TestingSystem/DVM/DVMTestingPlanner.java
View File

@@ -1,24 +1,31 @@
package TestingSystem.DVM;
import Common.Constants;
import Common.Global;
import Common.Utils.Utils;
import GlobalData.Machine.Machine;
import GlobalData.Machine.MachineType;
import GlobalData.RemoteFile.RemoteFile;
import GlobalData.Tasks.TaskState;
import GlobalData.User.User;
import Repository.EmailMessage;
import ProjectData.Files.ProjectFile;
import ProjectData.LanguageName;
import Repository.Server.ServerCode;
import Repository.Server.ServerExchangeUnit_2021;
import TestingSystem.Common.TestingPackage.TestingPackage;
import TestingSystem.Common.TestingPlanner;
import TestingSystem.DVM.DVMPackage.DVMPackage;
import TestingSystem.DVM.DVMTasks.DVMCompilationTask;
import TestingSystem.DVM.DVMTasks.DVMRunTask;
import TestingSystem.DVM.DVMTasks.DVMTask;
import TestingSystem.DVM.TasksPackage.TasksPackageState;
import TestingSystem.DVM.UserConnection;
import Visual_DVM_2021.Passes.PassException;
import Visual_DVM_2021.Passes.Server.TestingSystemPass;
import Visual_DVM_2021.Passes.All.UnzipFolderPass;
import Visual_DVM_2021.Passes.SSH.ConnectionPass;
import javafx.util.Pair;
import org.apache.commons.io.FileUtils;
import java.io.FileWriter;
import java.io.Serializable;
import java.util.Date;
import java.util.LinkedHashMap;
public class DVMTestingPlanner {
import java.io.File;
import java.io.FileFilter;
import java.nio.charset.Charset;
import java.nio.file.Paths;
import java.util.*;
public class DVMTestingPlanner extends TestingPlanner<DVMPackage> {
//----
LinkedHashMap<String, Machine> machines = new LinkedHashMap<>();
//todo приделать к ним очередь? или сделать в бд тестирования список машин.
@@ -27,164 +34,332 @@ public class DVMTestingPlanner {
Machine machine = null;
User user = null;
//----
DVMPackage dvmPackage; // текущий активный пакет тестирования DVM
int getSleepMillis() {
return 2000;
protected RemoteFile packageRemoteWorkspace;
protected File packageLocalWorkspace;
//--
public String getPlanner() {
return String.join("/", user.workspace, ConnectionPass.modules, ConnectionPass.planner);
}
//--
//Получить ид тестов и их папки на сервере.
LinkedHashMap<Integer, File> getTestsFromJson() {
LinkedHashMap<Integer, File> res = new LinkedHashMap<>();
for (DVMCompilationTask task : testingPackage.package_json.compilationTasks) {
if (!res.containsKey(task.test_id)) {
res.put(task.test_id, Paths.get(Global.TestsDirectory.getAbsolutePath(), String.valueOf(task.test_id)).toFile());
}
}
return res;
}
static LinkedHashMap<LanguageName, Vector<ProjectFile>> getTestPrograms(File test) {
LinkedHashMap<LanguageName, Vector<ProjectFile>> res = new LinkedHashMap<>();
//--
res.put(LanguageName.fortran, new Vector<>());
res.put(LanguageName.c, new Vector<>());
res.put(LanguageName.cpp, new Vector<>());
//--
File[] files = test.listFiles(new FileFilter() {
@Override
public boolean accept(File pathname) {
return pathname.isFile();
}
});
if (files != null) {
for (File file : files) {
ProjectFile projectFile = new ProjectFile(new File(file.getName()));
if (projectFile.isNotExcludedProgram())
res.get(projectFile.languageName).add(projectFile);
}
}
return res;
}
static void generateForLanguage(
String dvm_drv,
LanguageName language,
Vector<ProjectFile> language_programs,
Vector<String> titles,
Vector<String> objects,
Vector<String> bodies,
String flags
) {
if (!language_programs.isEmpty()) {
String LANG_ = language.toString().toUpperCase() + "_";
Vector<String> module_objects = new Vector<>();
String module_body = "";
int i = 1;
for (ProjectFile program : language_programs) {
//--
String object = Utils.DQuotes(language + "_" + i + ".o");
module_objects.add(object);
module_body +=
object + ":\n" +
"\t" +
String.join(" ",
Utils.MFVar(LANG_ + "COMMAND"),
Utils.MFVar(LANG_ + "FLAGS"),
program.getStyleOptions(),
"-c",
program.getQSourceName(),
"-o",
object + "\n\n"
);
++i;
}
titles.add(String.join("\n",
LANG_ + "COMMAND=" + Utils.DQuotes(dvm_drv) + " " +
language.getDVMCompile(),
LANG_ + "FLAGS=" + flags,
LANG_ + "OBJECTS=" + String.join(" ", module_objects),
""
));
objects.add(Utils.MFVar(LANG_ + "OBJECTS"));
bodies.add(module_body);
}
}
static String generateMakefile(File test, LanguageName test_language, String dvm_drv, String flags) {
//----->>
LinkedHashMap<LanguageName, Vector<ProjectFile>> programs = getTestPrograms(test);
Vector<String> titles = new Vector<>();
Vector<String> objects = new Vector<>();
Vector<String> bodies = new Vector<>();
String binary = Utils.DQuotes("0");
//----->>
for (LanguageName languageName : programs.keySet())
generateForLanguage(dvm_drv, languageName, programs.get(test_language), titles, objects, bodies, flags);
//----->>
return String.join("\n",
"LINK_COMMAND=" + Utils.DQuotes(dvm_drv) + " " +
test_language.getDVMLink(),
"LINK_FLAGS=" + flags + "\n",
String.join("\n", titles),
"all: " + binary,
binary + " : " + String.join(" ", objects),
"\t" + Utils.MFVar("LINK_COMMAND") + " " + Utils.MFVar("LINK_FLAGS") + " " + String.join(" ", objects) + " -o " + binary,
String.join(" ", bodies));
}
public void getTasksInfo(List<? extends DVMTask> tasks, String file_name) throws Exception {
LinkedHashMap<Integer, DVMTask> sorted_tasks = new LinkedHashMap<>();
for (DVMTask task : tasks)
sorted_tasks.put(task.id, task);
//--
File info_file = Paths.get(packageLocalWorkspace.getAbsolutePath(), "results", file_name).toFile();
List<String> lines = FileUtils.readLines(info_file, Charset.defaultCharset());
for (String packed : lines) {
if (!packed.isEmpty()) {
String[] data = packed.split(" ");
int id = Integer.parseInt(data[0]);
TaskState state = TaskState.valueOf(data[1]);
double time = Double.parseDouble(data[2]);
//--
DVMTask task = sorted_tasks.get(id);
task.state = state;
task.Time = state.equals(TaskState.AbortedByTimeout) ? (task.maxtime + 1) : time;
}
}
}
//---
Object ServerCommand(ServerCode code_in, String arg, Serializable object_in) throws Exception {
TestingSystemPass<Object> pass = new TestingSystemPass<Object>() {
@Override
public String getDescription() {
return "";
protected ServerCode getActivePackageCode() {
return ServerCode.GetFirstActiveDVMPackage;
}
@Override
protected void ServerAction() throws Exception {
Command(new ServerExchangeUnit_2021(code_in, arg, object_in));
target = response.object;
protected ServerCode getCheckIfNeedsKillCode() {
return ServerCode.DVMPackageNeedsKill;
}
};
if (!pass.Do()) throw new PassException("Ошибка взаимодействия с сервером " + code_in);
return pass.target;
@Override
protected TasksPackageState getStateAfterStart() {
return TasksPackageState.CompilationWorkspacesCreation;
}
Object ServerCommand(ServerCode code_in, Serializable object_in) throws Exception {
return ServerCommand(code_in, "", object_in);
@Override
protected void TestsSynchronize() throws Exception {
testingPackage.readJson();
LinkedHashMap<Integer, File> tests = getTestsFromJson();
//синхронизировать их.
for (int test_id : tests.keySet()) {
File test = tests.get(test_id);
RemoteFile test_dst = new RemoteFile(testingPackage.user_workspace + "/projects/" + test_id, true);
user.connection.MKDIR(test_dst);
user.connection.SynchronizeSubDirsR(test, test_dst);
}
Object ServerCommand(ServerCode code_in) throws Exception {
return ServerCommand(code_in, "", null);
}
//---
boolean isPrintOn() {
@Override
protected void PackageWorkspaceCreation() throws Exception {
testingPackage.readJson();
//--
LinkedHashMap<Integer, File> tests = getTestsFromJson();
//создать папку для пакета.
user.connection.sftpChannel.mkdir(packageRemoteWorkspace.full_name);
//положить туда запакованные тексты задач.
Vector<String> compilationLines = new Vector<>();
Vector<String> runLines = new Vector<>();
for (DVMCompilationTask compilationTask : testingPackage.package_json.compilationTasks) {
String makefileText = generateMakefile(tests.get(compilationTask.test_id), compilationTask.language, testingPackage.drv, compilationTask.flags);
compilationLines.addAll(compilationTask.pack(makefileText));
for (DVMRunTask runTask : compilationTask.runTasks)
runLines.addAll(runTask.pack(null));
}
RemoteFile compilationPackage = new RemoteFile(packageRemoteWorkspace, "compilationTasks");
RemoteFile runPackage = new RemoteFile(packageRemoteWorkspace, "runTasks");
user.connection.writeToFile(String.join("\n", compilationLines) + "\n", compilationPackage);
user.connection.writeToFile(String.join("\n", runLines) + "\n", runPackage);
// --
user.connection.MKDIR(new RemoteFile(packageRemoteWorkspace, "state"));
}
@Override
protected void PackageStart() throws Exception {
user.connection.ShellCommand("ulimit -s unlimited"); // нужно, для запуска сишной части.
String plannerStartCommand =
String.join(" ",
"nohup",
Utils.DQuotes(getPlanner()),
Utils.DQuotes(user.workspace),
Utils.DQuotes(packageRemoteWorkspace.full_name),
Utils.DQuotes(testingPackage.kernels),
Utils.DQuotes(testingPackage.drv),
"&"
);
user.connection.ShellCommand(plannerStartCommand);
RemoteFile PID = new RemoteFile(packageRemoteWorkspace, "PID");
RemoteFile STARTED = new RemoteFile(packageRemoteWorkspace, "STARTED");
while (!user.connection.Exists(STARTED)) {
Print("waiting for package start...");
Utils.sleep(1000);
}
if (user.connection.Exists(PID)) {
testingPackage.PID = user.connection.readFromFile(PID);
}
}
@Override
protected boolean CheckNextState() throws Exception {
RemoteFile stateDir = new RemoteFile(packageRemoteWorkspace, "state");
//состояния пакета могут меняться только по возрастанию. ищем, появилось ли такое.
Vector<TasksPackageState> higherStates = testingPackage.state.getHigherStates();
Collections.reverse(higherStates); //берем в обратном порядке, чтобы быстрее найти высшее.
for (TasksPackageState state : higherStates) {
RemoteFile file = new RemoteFile(stateDir, state.toString());
if (user.connection.Exists(file)) {
Print("found new state: " + file.name);
return true;
}
public void Print(String message) {
try {
if (isPrintOn()) {
FileWriter testLog = new FileWriter(getClass().getSimpleName() + "_Log.txt", true);
String dmessage = Utils.Brackets(new Date()) + " " + message;
System.out.println(dmessage);
testLog.write(dmessage + "\n");
testLog.close();
}
} catch (Exception ex) {
ex.printStackTrace();
}
return false;
}
@Override
protected void DownloadResults() throws Exception {
Utils.CheckDirectory(packageLocalWorkspace);
RemoteFile remote_results_archive = new RemoteFile(packageRemoteWorkspace, "results.zip");
File results_archive = new File(packageLocalWorkspace, "results.zip");
user.connection.performScript(packageRemoteWorkspace, "zip -r " + Utils.DQuotes("results.zip") + " " + Utils.DQuotes("results"));
//---
void CheckMachineAndUser() {
String machine_url = dvmPackage.machine_address + ":" + dvmPackage.machine_port;
if (!machines.containsKey(machine_url))
machines.put(machine_url, new Machine(
dvmPackage.machine_name,
dvmPackage.machine_address,
dvmPackage.machine_port,
MachineType.Server));
if (!users.containsKey(dvmPackage.user_name))
users.put(dvmPackage.user_name,
new User(dvmPackage.user_name, dvmPackage.user_password, dvmPackage.user_workspace));
//-->>
machine = machines.get(machine_url);
user = users.get(dvmPackage.user_name);
if (user.connection.Exists(remote_results_archive)) {
user.connection.getSingleFile(remote_results_archive.full_name, results_archive.getAbsolutePath());
UnzipFolderPass unzipFolderPass = new UnzipFolderPass();
unzipFolderPass.Do(results_archive.getAbsolutePath(), packageLocalWorkspace.getAbsolutePath(), false);
}
//todo привязать это к настройкам, чтобы можно было включать/выключать удаление.
/*
//получили результат. теперь уничтожаем папку пакета на целевой машине.
if ( user.connection.Exists(packageRemoteWorkspace)){
user.connection.RMDIR(packageRemoteWorkspace.full_name);
}
*/
}
@Override
protected void AnalyseResults() throws Exception {
testingPackage.readJson();
Print("analysing results");
Vector<DVMRunTask> runTasks = new Vector<>();
for (DVMCompilationTask compilationTask : testingPackage.package_json.compilationTasks)
runTasks.addAll(compilationTask.runTasks);
//----
getTasksInfo(testingPackage.package_json.compilationTasks, "CompilationInfo.txt");
getTasksInfo(runTasks, "RunningInfo.txt");
//--
int ct_count = 0;
int rt_count = 0;
//--
for (DVMCompilationTask compilationTask : testingPackage.package_json.compilationTasks) {
ct_count++;
File ct_workspace = Paths.get(packageLocalWorkspace.getAbsolutePath(), "results", String.valueOf(compilationTask.id)).toFile();
if (ct_workspace.exists()) {
for (DVMRunTask runTask : compilationTask.runTasks) {
rt_count++;
runTask.compilation_state = compilationTask.state;
runTask.compilation_time = compilationTask.Time;
if (compilationTask.state == TaskState.DoneWithErrors) {
runTask.state = TaskState.Canceled;
} else {
File rt_workspace = Paths.get(packageLocalWorkspace.getAbsolutePath(), "results", String.valueOf(runTask.id)).toFile();
if (rt_workspace.exists() && runTask.state.equals(TaskState.Finished)) {
//анализ задачи на запуск.
File outFile = new File(rt_workspace, Constants.out_file);
File errFile = new File(rt_workspace.getAbsolutePath(), Constants.err_file);
//--
String output = FileUtils.readFileToString(outFile);
String errors = FileUtils.readFileToString(errFile);
//--
List<String> output_lines = Arrays.asList(output.split("\n"));
List<String> errors_lines = Arrays.asList(errors.split("\n"));
//---
if (Utils.isCrushed(output_lines, errors_lines)) {
runTask.state = TaskState.Crushed;
} else {
Pair<TaskState, Integer> results = new Pair<>(TaskState.Done, 100);
switch (runTask.test_type) {
case Correctness:
results = Utils.analyzeCorrectness(output_lines);
break;
case Performance:
results = Utils.analyzePerformance(output_lines);
break;
default:
break;
}
runTask.state = results.getKey();
runTask.progress = results.getValue();
runTask.CleanTime = Utils.parseCleanTime(output);
}
}
}
}
testingPackage.saveJson(); //запись обновленных результатов пакета в json!
Print("analysis done, ct_count=" + ct_count + " rt count=" + rt_count);
//--
}
boolean CheckConnection(Machine machine, User user) {
//каждый раз соединяемся по новой. из за проблем с Exists.
// к тому же, теперь задачи гоняет модуль, тут только проверка
//так что время на разрыв уже не критично.
//todo попробовать все же вариант с постоянным соединением.
}
}
@Override
protected void Kill() throws Exception {
if (!testingPackage.PID.isEmpty()) {
user.connection.ShellCommand("kill -9 " + testingPackage.PID);
}
}
@Override
protected void InitSessionCredentials() {
String machine_url = testingPackage.machine_address + ":" + testingPackage.machine_port;
if (!machines.containsKey(machine_url))
machines.put(machine_url, testingPackage.getMachine());
if (!users.containsKey(testingPackage.user_name))
users.put(testingPackage.user_name, testingPackage.getUser());
//-->>
machine = machines.get(machine_url);
user = users.get(testingPackage.user_name);
//--
packageRemoteWorkspace = new RemoteFile(user.workspace + "/tests", String.valueOf(testingPackage.id), true);
packageLocalWorkspace = new File(Global.PackagesDirectory, String.valueOf(testingPackage.id));
}
@Override
protected boolean Connect() {
try {
user.connection = null;
user.connection = new UserConnection(machine, user);
Print("Соединение c " + machine.getURL() + " " + user.login + " успешно установлено.");
user.connection.ShellCommand("ulimit -s unlimited"); // нужно, для запуска сишной части.
} catch (Exception ex) {
Global.Log.PrintException(ex);
Print(ex.toString());
user.connection = null;
Print("Не удалось установить соединение.");
}
return user.connection != null;
}
//--
void EmailPackage(TestingPackage package_in) throws Exception{
EmailMessage message = new EmailMessage();
message.subject = "Состояние пакета задач " + Utils.Brackets(package_in) + " изменилось на " + Utils.Brackets(package_in.state.getDescription());
message.text = package_in.description;
message.targets.add(package_in.sender_address);
ServerCommand(ServerCode.Email, message);
}
void UpdatePackage(TestingPackage package_in){
}
// состояния пакета.
void AnalyseResults() {
}
//---
public void Perform() {
/*
while (true) {
try {
dvmPackage = (DVMPackage) ServerCommand(ServerCode.GetFirstActiveDVMPackage);
if (dvmPackage != null) {
System.out.println(dvmPackage.id + " " + dvmPackage.state.getDescription());
//--
CheckMachineAndUser();
if (dvmPackage.state.equals(TasksPackageState.Analysis)) {
AnalyseResults(); //на анализ уже не нужно соединение.
dvmPackage.state = TasksPackageState.Done;
UpdateDVMPackage();
} else {
if (CheckConnection(machine, user)) {
try {
switch (dvmPackage.state){
case TestsSynchronize:
TestsSynchronize();
dvmPackage.state = TasksPackageState.PackageWorkspaceCreation;
UpdateDVMPackage();
break;
case PackageWorkspaceCreation:
PackageWorkspaceCreation();
tasksPackage.state = TasksPackageState.PackageStart;
UpdateDVMPackage();
break;
case PackageStart:
PackageStart();
tasksPackage.state = TasksPackageState.CompilationWorkspacesCreation;
UpdateDVMPackage();
break;
case CompilationWorkspacesCreation:
case CompilationPreparation:
case CompilationExecution:
case RunningWorkspacesCreation:
case RunningPreparation:
case RunningExecution:
checkNextState();
break;
case RunningEnd:
DownloadResults();
dvmPackage.state = TasksPackageState.Analysis;
planner.UpdatePackage();
break;
}
} catch (Exception ex) {
Print("Ошибка сеанса.");
Print(ex.getMessage());
}
finally {
if (user.connection != null) {
user.connection.Disconnect();
user.connection = null;
}
}
}
}
}
} catch (Exception ex) {
ex.printStackTrace();
} finally {
Utils.sleep(getSleepMillis());
}
}
*/
}
}