Merge pull request #1399 from drgrice1/asksage-update

Update the `AskSage` method.

Author: somiaj

.gitignore

@@ -10,3 +10,5 @@ htdocs/static-assets.json
 htdocs/**/*.min.js
 htdocs/**/*.min.css
 htdocs/tmp
+
+sagecell-docker/sagecell-docker.service

conf/pg_config.dist.yml

@@ -74,6 +74,17 @@ URLs:
     - .
     - $pg_root_url/images
 
+  # By default the AskSage method uses the public sagecell service located at
+  # https://sagecell.sagemath.org/service. However, the server at that address
+  # is no longer accepting public requests. So in order for problems that use
+  # the AskSage method to work a different sagecell server must be used.
+  # Uncomment the line below and set the value to a custom sagecell server
+  # address to use with the AskSage method.  If using the docker container built
+  # with the instructions and files in the sagecell-docker directory in this
+  # repository, the use the line below as is.
+
+  #sagecellService: http://localhost:3500/service
+
 # Flat-file database used to protect against MD5 hash collisions. TeX equations
 # are hashed to determine the name of the image file. There is a tiny chance of
 # a collision between two TeX strings. This file allows for that. However, this

lib/PGcore.pm

@@ -703,9 +703,8 @@ sub directoryFromPath {
 }
 
 sub AskSage {
-	my $self    = shift;
-	my $python  = shift;
-	my $options = shift;
+	my ($self, $python, $options) = @_;
+	$options = {} unless ref $options eq 'HASH';
 	$options->{curlCommand} = WeBWorK::PG::IO::externalCommand('curl');
 	WeBWorK::PG::IO::AskSage($python, $options);
 }

lib/WeBWorK/PG/IO.pm

@@ -315,7 +315,7 @@ sub externalCommand {
 
 # Isolate the call to the sage server in case we have to jazz it up.
 sub query_sage_server {
-	my ($python, $url, $accepted_tos, $setSeed, $webworkfunc, $debug, $curlCommand) = @_;
+	my ($python, $url, $setSeed, $webworkfunc, $debug, $curlCommand) = @_;
 
 	# Validate url to prevent shell injection — must look like a URL.
 	if ($url && $url !~ m{^https?://[^\s;|&`\$]+$}) {
@@ -328,7 +328,7 @@ sub query_sage_server {
 		$curlCommand,                                                 '-i',
 		'-k',                                                         '-sS',
 		'-L',                                                         '--data-urlencode',
-		"accepted_tos=$accepted_tos",                                 '--data-urlencode',
+		"accepted_tos=true",                                          '--data-urlencode',
 		'user_expressions={"WEBWORK":"_webwork_safe_json(WEBWORK)"}', '--data-urlencode',
 		"code=${setSeed}${webworkfunc}$python",                       $url // (),
 	);
@@ -400,8 +400,7 @@ sub query_sage_server {
 		# Success! Put any extraneous splits back together.
 		$result = join("\r\n\r\n", @lines);
 	} else {
-		warn "ERROR in contacting sage server. Did you accept the terms of service by "
-			. "setting { accepted_tos => 'true' } in the askSage options?\n$content\n";
+		warn "ERROR in contacting sage server.\n$content\n";
 		$result = undef;
 	}
 
@@ -421,44 +420,44 @@ sub AskSage {
 	chomp($python);
 
 	# To send values back in a hash, add them to the python WEBWORK dictionary.
-	my $url          = $args->{url} || 'https://sagecell.sagemath.org/service';
-	my $seed         = $args->{seed};
-	my $accepted_tos = $args->{accepted_tos} || 'false';    # Force author to accept terms of service explicitly.
-	my $debug        = $args->{debug}        || 0;
-	my $setSeed      = $seed ? "set_random_seed($seed)\n" : '';
-	my $curlCommand  = $args->{curlCommand};
+	my $url         = $args->{url} || $pg_envir->{URLs}{sagecellService} || 'https://sagecell.sagemath.org/service';
+	my $seed        = $args->{seed};
+	my $debug       = $args->{debug} || 0;
+	my $setSeed     = $seed ? "set_random_seed($seed)\n" : '';
+	my $curlCommand = $args->{curlCommand};
 
 	my $webworkfunc = <<END;
 WEBWORK={}
 
 def _webwork_safe_json(o):
     import json
-    def default(o):
-        try:
-            if isinstance(o,sage.rings.integer.Integer):
-                json_obj = int(o)
-            elif isinstance(o,(sage.rings.real_mpfr.RealLiteral, sage.rings.real_mpfr.RealNumber)):
-                json_obj = float(o)
-            elif sage.modules.free_module_element.is_FreeModuleElement(o):
-                json_obj = list(o)
-            elif sage.matrix.matrix.is_Matrix(o):
-                json_obj = [list(i) for i in o.rows()]
-            elif isinstance(o, SageObject):
-                json_obj = repr(o)
+    class WebworkEncoder(json.JSONEncoder):
+        def default(self, o):
+            try:
+                if isinstance(o,sage.rings.integer.Integer):
+                    json_obj = int(o)
+                elif isinstance(o,(sage.rings.real_mpfr.RealLiteral, sage.rings.real_mpfr.RealNumber)):
+                    json_obj = float(o)
+                elif isinstance(o, sage.modules.free_module_element.FreeModuleElement):
+                    json_obj = list(o)
+                elif isinstance(o, sage.matrix.matrix0.Matrix):
+                    json_obj = [list(i) for i in o.rows()]
+                elif isinstance(o, SageObject):
+                    json_obj = repr(o)
+                else:
+                    raise TypeError
+            except TypeError:
+                pass
             else:
-                raise TypeError
-        except TypeError:
-            pass
-        else:
-            return json_obj
-        # Let the base class default method raise the TypeError
-        return json.JSONEncoder.default(self, o)
-    return json.dumps(o, default=default)
+                return json_obj
+            # Let the base class default method raise the TypeError
+            return super().default(o)
+    return json.dumps(o, cls=WebworkEncoder)
 END
 
 	my $ret = { success => 0 };    # We want to export more than one piece of information.
 	eval {
-		my $output = query_sage_server($python, $url, $accepted_tos, $setSeed, $webworkfunc, $debug, $curlCommand);
+		my $output = query_sage_server($python, $url, $setSeed, $webworkfunc, $debug, $curlCommand);
 
 		# has something been returned?
 		not_null($output) or die "Unable to make a sage call to $url.";
@@ -486,7 +485,7 @@ END
 		warn "now success  is $success because status was ok" if $debug;
 		if ($success) {
 			my $WEBWORK_variable_non_empty = 0;
-			my $sage_WEBWORK_data          = $decoded->{execute_reply}{user_expressions}{WEBWORK}{data}{'text/plain'};
+			my $sage_WEBWORK_data = $decoded->{execute_reply}{user_expressions}{WEBWORK}{data}{'text/plain'} // '';
 			warn "sage_WEBWORK_data $sage_WEBWORK_data" if $debug;
 			if (not_null($sage_WEBWORK_data)) {
 				$WEBWORK_variable_non_empty =    # another hack because '{}' is sometimes returned

macros/PG.pl

@@ -397,9 +397,7 @@ ()
 }
 
 sub AskSage {
-	my $python  = shift;
-	my $options = shift;
-	WARN_MESSAGE("the second argument to AskSage should be a hash of options") unless $options =~ /HASH/;
+	my ($python, $options) = @_;
 	$PG->AskSage($python, $options);
 }
 

sagecell-docker/Dockerfile

@@ -0,0 +1,63 @@
+FROM sagemath/sagemath:10.8
+
+USER root
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV DEBCONF_NONINTERACTIVE_SEEN=true
+ENV DEBCONF_NOWARNINGS=yes
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    automake \
+    build-essential \
+    curl \
+    git \
+    locales \
+    openssh-server \
+    python3 \
+    python-is-python3 \
+    python3-jupyter-client \
+    python3-pip \
+    python3-psutil \
+    python3-zmq \
+    rsyslog \
+    wget \
+    && echo "en_US.UTF-8 UTF-8" > /etc/locale.gen \
+    && locale-gen \
+    && curl -fsSL https://deb.nodesource.com/setup_24.x | bash - \
+    && apt-get install -y --no-install-recommends --no-install-suggests nodejs
+
+ENV SHELL=/bin/bash
+
+USER sage
+
+RUN echo "Installing SageCell server for SageMath" \
+    && sage -pip install lockfile paramiko sockjs-tornado sqlalchemy \
+    && git clone --single-branch --branch master --depth 1 https://github.com/sagemath/sagecell.git \
+    && cd sagecell \
+    && { sage -sh -c make || true; } \
+    && sage -sh -c make \
+    && rm -rf contrib doc tests .git \
+    && cp config_default.py config.py \
+    && sed -i 's/"username": None/"username": "sage"/' config.py \
+	&& sed -i 's/requires_tos = True/requires_tos = False/' config.py
+
+USER root
+
+COPY sagecell-entrypoint.sh /usr/local/bin/sagecell-entrypoint
+
+RUN echo "Configuring internal ssh server" \
+    && chmod +x /usr/local/bin/sagecell-entrypoint \
+    && echo "ListenAddress 127.0.0.1\nPasswordAuthentication no" \
+        > /etc/ssh/sshd_config.d/sshd_config.conf
+
+RUN echo "Cleaning the Container" \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/* \
+    && rm -rf /tmp/*
+
+ENV PORT=3500
+
+WORKDIR /home/sage/sagecell
+
+ENTRYPOINT ["sagecell-entrypoint"]
+CMD ["sudo", "-H", "-E", "-u", "sage", "sage", "web_server.py", "-p $PORT"]

sagecell-docker/README.md

@@ -0,0 +1,47 @@
+# SageCell Docker Instructions
+
+These are instructions to build and use a docker image for running a SageCell service in production.
+
+## Build and Run the Docker Image
+
+Execute the following command from the `sagecell-docker` directory.
+
+```bash
+docker build -t sagecell .
+```
+
+Then run the container by executing the following command in the `sagecell-docker` directory.
+
+```bash
+docker run --rm -p 127.0.0.1:3500:3500 sagecell:latest
+```
+
+Note that the `-p` argument exposes port `3500` inside the container to port `3500` outside the container, but only
+allows connections from localhost. This is important as it does not allow anyone else to use the SageCell service other
+than the local webwork server. If a different port outside the container is needed, then change the first `3500` to the
+desired port. Adding `--rm` removes the created volume for the container when it exits. Exit the container by typing
+`Ctrl-C`.
+
+Access by specific external IP addresses is also possible if you want to have the SageCell service on another server.
+The easiest way to accomplish this is by using an SSH tunnel. For example, execute
+`ssh -L 3500:127.0.0.1:3550 userId@sagecell.server` where `userId` is your username on the SageCell server and
+`sagecell.server` is the domain name or IP address of the SageCell server. There are other ways to accomplish this as
+well, but the important thing is that access to the SageCell service be highly restricted as it allows untrusted code to
+be executed.
+
+## Deploy the Image for Production Usage
+
+Copy the file `sagecell-docker.dist.service` to `sagecell-docker.service` and execute the following command from the
+`sagecell-docker` directory to enable a `systemd` service for running the container.
+
+```bash
+sudo systemctl enable $(pwd)/sagecell-docker.service
+```
+
+Then you can start the container by executing `sudo systemctl start sagecell-docker`, and stop the container by
+executing `sudo systemctl stop sagecell-docker`. Note that any time the server is rebooted the service will start
+automatically, so you usually do not need to execute those commands other than to start the service the first time after
+enabling it.
+
+Note that the service simply executes the command give above for running the container directly. So modify the command
+in the `sagecell-docker.service` file as described above if a different port is needed.

sagecell-docker/sagecell-docker.dist.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=sagecell server
+After=network.target
+
+[Service]
+Type=simple
+ExecStart=docker run --rm -p 127.0.0.1:3500:3500 sagecell:latest
+KillMode=process
+
+[Install]
+WantedBy=multi-user.target

sagecell-docker/sagecell-entrypoint.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+
+function configure_ssh() {
+	local key_file=/home/sage/.ssh/id_rsa
+	if [ $(ssh-add -l &> /dev/null; echo $?) -gt 0 ]; then
+		# Generate ssh keys for the user.
+		if [ ! -f $key_file ]; then
+			[ ! -d /home/sage/.ssh ] && mkdir /home/sage/.ssh
+			ssh-keygen -t rsa -f $key_file -P "" -q
+		fi
+
+		# Start and configure the ssh-agent.
+		eval `ssh-agent` > /dev/null
+		local ssh_fingerprint=$(ssh-keygen -l -f $key_file | awk '{print $2}')
+		if [ ! -z  "$ssh_fingerprint" ] && [ -z "$(ssh-add -l | grep "$ssh_fingerprint")" ]; then
+			ssh-add $key_file 2> /dev/null
+		fi
+
+		# Add localhost to known_hosts.
+		[ ! -f /home/sage/.ssh/known_hosts ] && touch /home/sage/.ssh/known_hosts
+		if [ -z "$(ssh-keygen -F localhost)" ]; then
+			ssh-keyscan -H localhost > /home/sage/.ssh/known_hosts 2> /dev/null
+		fi
+
+		cat ${key_file}.pub > /home/sage/.ssh/authorized_keys
+	fi
+}
+
+function check_ssh() {
+	ssh -v -o PreferredAuthentications=publickey -o BatchMode=yes -o ConnectTimeout=10 \
+		-o StrictHostKeyChecking=no localhost /bin/true &> /dev/null
+	return $?
+}
+
+# Configure and start rsyslog.
+sed -i '/imklog/s/^/#/' /etc/rsyslog.conf
+rsyslogd
+
+# Start and configure the ssh server for internal use.
+service ssh start > /dev/null
+
+export -f configure_ssh
+export -f check_ssh
+
+su sage bash -c "configure_ssh"
+su sage bash -c "check_ssh"
+
+if [ $? -eq 0 ]; then
+    echo "Executing: $(eval "echo $@")"
+    exec $(eval "echo $@")
+else
+    echo "ERROR: SageCell server not started. Failed to configure internal ssh server."
+fi